VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51731

Last change on this file since 51731 was 51685, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Nit.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 491.4 KB
Line 
1/* $Id: HMVMXR0.cpp 51685 2014-06-23 03:45:00Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#include <VBox/vmm/gim.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#ifdef DEBUG_ramshankar
41# define HMVMX_SAVE_FULL_GUEST_STATE
42# define HMVMX_SYNC_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_CHECK_GUEST_STATE
44# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
45# define HMVMX_ALWAYS_TRAP_PF
46# define HMVMX_ALWAYS_SWAP_FPU_STATE
47# define HMVMX_ALWAYS_FLUSH_TLB
48# define HMVMX_ALWAYS_SWAP_EFER
49#endif
50
51
52/*******************************************************************************
53* Defined Constants And Macros *
54*******************************************************************************/
55#if defined(RT_ARCH_AMD64)
56# define HMVMX_IS_64BIT_HOST_MODE() (true)
57typedef RTHCUINTREG HMVMXHCUINTREG;
58#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
59extern "C" uint32_t g_fVMXIs64bitHost;
60# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
61typedef uint64_t HMVMXHCUINTREG;
62#else
63# define HMVMX_IS_64BIT_HOST_MODE() (false)
64typedef RTHCUINTREG HMVMXHCUINTREG;
65#endif
66
67/** Use the function table. */
68#define HMVMX_USE_FUNCTION_TABLE
69
70/** Determine which tagged-TLB flush handler to use. */
71#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
72#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
73#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
74#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
75
76/** @name Updated-guest-state flags.
77 * @{ */
78#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
79#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
80#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
81#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
82#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
83#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
84#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
85#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
86#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
87#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
88#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
89#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
90#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
91#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
92#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
93#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
94#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
95#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
96#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
97#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
98 | HMVMX_UPDATED_GUEST_RSP \
99 | HMVMX_UPDATED_GUEST_RFLAGS \
100 | HMVMX_UPDATED_GUEST_CR0 \
101 | HMVMX_UPDATED_GUEST_CR3 \
102 | HMVMX_UPDATED_GUEST_CR4 \
103 | HMVMX_UPDATED_GUEST_GDTR \
104 | HMVMX_UPDATED_GUEST_IDTR \
105 | HMVMX_UPDATED_GUEST_LDTR \
106 | HMVMX_UPDATED_GUEST_TR \
107 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
108 | HMVMX_UPDATED_GUEST_DEBUG \
109 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
110 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
112 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
113 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
114 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
115 | HMVMX_UPDATED_GUEST_APIC_STATE)
116/** @} */
117
118/** @name
119 * Flags to skip redundant reads of some common VMCS fields that are not part of
120 * the guest-CPU state but are in the transient structure.
121 */
122#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
123#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
129/** @} */
130
131/** @name
132 * States of the VMCS.
133 *
134 * This does not reflect all possible VMCS states but currently only those
135 * needed for maintaining the VMCS consistently even when thread-context hooks
136 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
137 */
138#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
139#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
140#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
141/** @} */
142
143/**
144 * Exception bitmap mask for real-mode guests (real-on-v86).
145 *
146 * We need to intercept all exceptions manually (except #PF). #NM is also
147 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
148 * even in real-mode if we have Nested Paging support.
149 */
150#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
151 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
152 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
153 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
154 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
155 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
156 | RT_BIT(X86_XCPT_XF))
157
158/**
159 * Exception bitmap mask for all contributory exceptions.
160 *
161 * Page fault is deliberately excluded here as it's conditional as to whether
162 * it's contributory or benign. Page faults are handled separately.
163 */
164#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
165 | RT_BIT(X86_XCPT_DE))
166
167/** Maximum VM-instruction error number. */
168#define HMVMX_INSTR_ERROR_MAX 28
169
170/** Profiling macro. */
171#ifdef HM_PROFILE_EXIT_DISPATCH
172# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
173# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
174#else
175# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
176# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
177#endif
178
179/** Assert that preemption is disabled or covered by thread-context hooks. */
180#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
181 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
182
183/** Assert that we haven't migrated CPUs when thread-context hooks are not
184 * used. */
185#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
186 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
187 ("Illegal migration! Entered on CPU %u Current %u\n", \
188 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
189
190/** Helper macro for VM-exit handlers called unexpectedly. */
191#define HMVMX_RETURN_UNEXPECTED_EXIT() \
192 do { \
193 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
194 return VERR_VMX_UNEXPECTED_EXIT; \
195 } while (0)
196
197
198/*******************************************************************************
199* Structures and Typedefs *
200*******************************************************************************/
201/**
202 * VMX transient state.
203 *
204 * A state structure for holding miscellaneous information across
205 * VMX non-root operation and restored after the transition.
206 */
207typedef struct VMXTRANSIENT
208{
209 /** The host's rflags/eflags. */
210 RTCCUINTREG uEflags;
211#if HC_ARCH_BITS == 32
212 uint32_t u32Alignment0;
213#endif
214 /** The guest's TPR value used for TPR shadowing. */
215 uint8_t u8GuestTpr;
216 /** Alignment. */
217 uint8_t abAlignment0[7];
218
219 /** The basic VM-exit reason. */
220 uint16_t uExitReason;
221 /** Alignment. */
222 uint16_t u16Alignment0;
223 /** The VM-exit interruption error code. */
224 uint32_t uExitIntErrorCode;
225 /** The VM-exit exit qualification. */
226 uint64_t uExitQualification;
227
228 /** The VM-exit interruption-information field. */
229 uint32_t uExitIntInfo;
230 /** The VM-exit instruction-length field. */
231 uint32_t cbInstr;
232 /** The VM-exit instruction-information field. */
233 union
234 {
235 /** Plain unsigned int representation. */
236 uint32_t u;
237 /** INS and OUTS information. */
238 struct
239 {
240 uint32_t u6Reserved0 : 7;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u5Reserved1 : 5;
244 /** The segment register (X86_SREG_XXX). */
245 uint32_t iSegReg : 3;
246 uint32_t uReserved2 : 14;
247 } StrIo;
248 } ExitInstrInfo;
249 /** Whether the VM-entry failed or not. */
250 bool fVMEntryFailed;
251 /** Alignment. */
252 uint8_t abAlignment1[3];
253
254 /** The VM-entry interruption-information field. */
255 uint32_t uEntryIntInfo;
256 /** The VM-entry exception error code field. */
257 uint32_t uEntryXcptErrorCode;
258 /** The VM-entry instruction length field. */
259 uint32_t cbEntryInstr;
260
261 /** IDT-vectoring information field. */
262 uint32_t uIdtVectoringInfo;
263 /** IDT-vectoring error code. */
264 uint32_t uIdtVectoringErrorCode;
265
266 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
267 uint32_t fVmcsFieldsRead;
268
269 /** Whether the guest FPU was active at the time of VM-exit. */
270 bool fWasGuestFPUStateActive;
271 /** Whether the guest debug state was active at the time of VM-exit. */
272 bool fWasGuestDebugStateActive;
273 /** Whether the hyper debug state was active at the time of VM-exit. */
274 bool fWasHyperDebugStateActive;
275 /** Whether TSC-offsetting should be setup before VM-entry. */
276 bool fUpdateTscOffsettingAndPreemptTimer;
277 /** Whether the VM-exit was caused by a page-fault during delivery of a
278 * contributory exception or a page-fault. */
279 bool fVectoringPF;
280} VMXTRANSIENT;
281AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
285AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
286/** Pointer to VMX transient state. */
287typedef VMXTRANSIENT *PVMXTRANSIENT;
288
289
290/**
291 * MSR-bitmap read permissions.
292 */
293typedef enum VMXMSREXITREAD
294{
295 /** Reading this MSR causes a VM-exit. */
296 VMXMSREXIT_INTERCEPT_READ = 0xb,
297 /** Reading this MSR does not cause a VM-exit. */
298 VMXMSREXIT_PASSTHRU_READ
299} VMXMSREXITREAD;
300/** Pointer to MSR-bitmap read permissions. */
301typedef VMXMSREXITREAD* PVMXMSREXITREAD;
302
303/**
304 * MSR-bitmap write permissions.
305 */
306typedef enum VMXMSREXITWRITE
307{
308 /** Writing to this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
310 /** Writing to this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_WRITE
312} VMXMSREXITWRITE;
313/** Pointer to MSR-bitmap write permissions. */
314typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
315
316
317/**
318 * VMX VM-exit handler.
319 *
320 * @returns VBox status code.
321 * @param pVCpu Pointer to the VMCPU.
322 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
323 * out-of-sync. Make sure to update the required
324 * fields before using them.
325 * @param pVmxTransient Pointer to the VMX-transient structure.
326 */
327#ifndef HMVMX_USE_FUNCTION_TABLE
328typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329#else
330typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
331/** Pointer to VM-exit handler. */
332typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
333#endif
334
335
336/*******************************************************************************
337* Internal Functions *
338*******************************************************************************/
339static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
340static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
341static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
342 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
343#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
344static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
345#endif
346#ifndef HMVMX_USE_FUNCTION_TABLE
347DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
348# define HMVMX_EXIT_DECL static int
349#else
350# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
351#endif
352
353/** @name VM-exit handlers.
354 * @{
355 */
356static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
357static FNVMXEXITHANDLER hmR0VmxExitExtInt;
358static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
359static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
360static FNVMXEXITHANDLER hmR0VmxExitSipi;
361static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
362static FNVMXEXITHANDLER hmR0VmxExitSmi;
363static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
364static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
365static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
366static FNVMXEXITHANDLER hmR0VmxExitCpuid;
367static FNVMXEXITHANDLER hmR0VmxExitGetsec;
368static FNVMXEXITHANDLER hmR0VmxExitHlt;
369static FNVMXEXITHANDLER hmR0VmxExitInvd;
370static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
371static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
372static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
373static FNVMXEXITHANDLER hmR0VmxExitRsm;
374static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
375static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
376static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
377static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
378static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
379static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
380static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
381static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
382static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
383static FNVMXEXITHANDLER hmR0VmxExitMwait;
384static FNVMXEXITHANDLER hmR0VmxExitMtf;
385static FNVMXEXITHANDLER hmR0VmxExitMonitor;
386static FNVMXEXITHANDLER hmR0VmxExitPause;
387static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
388static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
389static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
390static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
391static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
392static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
393static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
394static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
395static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
396static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
397static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
398static FNVMXEXITHANDLER hmR0VmxExitRdrand;
399static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
400/** @} */
401
402static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
409static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410#endif
411static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
412
413/*******************************************************************************
414* Global Variables *
415*******************************************************************************/
416#ifdef HMVMX_USE_FUNCTION_TABLE
417
418/**
419 * VMX_EXIT dispatch table.
420 */
421static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
422{
423 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
424 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
425 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
426 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
427 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
428 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
429 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
430 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
431 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
432 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
433 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
434 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
435 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
436 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
437 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
438 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
439 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
440 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
441 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
442 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
443 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
444 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
445 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
446 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
447 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
448 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
449 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
450 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
451 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
452 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
453 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
454 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
455 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
456 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
457 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
458 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
459 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
460 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
461 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
462 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
463 /* 40 UNDEFINED */ hmR0VmxExitPause,
464 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
465 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
466 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
467 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
468 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
469 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
470 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
471 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
472 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
473 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
474 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
475 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
476 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
477 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
478 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
479 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
480 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
481 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
482 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
483};
484#endif /* HMVMX_USE_FUNCTION_TABLE */
485
486#ifdef VBOX_STRICT
487static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
488{
489 /* 0 */ "(Not Used)",
490 /* 1 */ "VMCALL executed in VMX root operation.",
491 /* 2 */ "VMCLEAR with invalid physical address.",
492 /* 3 */ "VMCLEAR with VMXON pointer.",
493 /* 4 */ "VMLAUNCH with non-clear VMCS.",
494 /* 5 */ "VMRESUME with non-launched VMCS.",
495 /* 6 */ "VMRESUME after VMXOFF",
496 /* 7 */ "VM entry with invalid control fields.",
497 /* 8 */ "VM entry with invalid host state fields.",
498 /* 9 */ "VMPTRLD with invalid physical address.",
499 /* 10 */ "VMPTRLD with VMXON pointer.",
500 /* 11 */ "VMPTRLD with incorrect revision identifier.",
501 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
502 /* 13 */ "VMWRITE to read-only VMCS component.",
503 /* 14 */ "(Not Used)",
504 /* 15 */ "VMXON executed in VMX root operation.",
505 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
506 /* 17 */ "VM entry with non-launched executing VMCS.",
507 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
508 /* 19 */ "VMCALL with non-clear VMCS.",
509 /* 20 */ "VMCALL with invalid VM-exit control fields.",
510 /* 21 */ "(Not Used)",
511 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
512 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
513 /* 24 */ "VMCALL with invalid SMM-monitor features.",
514 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
515 /* 26 */ "VM entry with events blocked by MOV SS.",
516 /* 27 */ "(Not Used)",
517 /* 28 */ "Invalid operand to INVEPT/INVVPID."
518};
519#endif /* VBOX_STRICT */
520
521
522
523/**
524 * Updates the VM's last error record. If there was a VMX instruction error,
525 * reads the error data from the VMCS and updates VCPU's last error record as
526 * well.
527 *
528 * @param pVM Pointer to the VM.
529 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
530 * VERR_VMX_UNABLE_TO_START_VM or
531 * VERR_VMX_INVALID_VMCS_FIELD).
532 * @param rc The error code.
533 */
534static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
535{
536 AssertPtr(pVM);
537 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
538 || rc == VERR_VMX_UNABLE_TO_START_VM)
539 {
540 AssertPtrReturnVoid(pVCpu);
541 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
542 }
543 pVM->hm.s.lLastError = rc;
544}
545
546
547/**
548 * Reads the VM-entry interruption-information field from the VMCS into the VMX
549 * transient structure.
550 *
551 * @returns VBox status code.
552 * @param pVmxTransient Pointer to the VMX transient structure.
553 *
554 * @remarks No-long-jump zone!!!
555 */
556DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
557{
558 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
559 AssertRCReturn(rc, rc);
560 return VINF_SUCCESS;
561}
562
563
564/**
565 * Reads the VM-entry exception error code field from the VMCS into
566 * the VMX transient structure.
567 *
568 * @returns VBox status code.
569 * @param pVmxTransient Pointer to the VMX transient structure.
570 *
571 * @remarks No-long-jump zone!!!
572 */
573DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
574{
575 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
576 AssertRCReturn(rc, rc);
577 return VINF_SUCCESS;
578}
579
580
581/**
582 * Reads the VM-entry exception error code field from the VMCS into
583 * the VMX transient structure.
584 *
585 * @returns VBox status code.
586 * @param pVmxTransient Pointer to the VMX transient structure.
587 *
588 * @remarks No-long-jump zone!!!
589 */
590DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
591{
592 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
593 AssertRCReturn(rc, rc);
594 return VINF_SUCCESS;
595}
596
597
598/**
599 * Reads the VM-exit interruption-information field from the VMCS into the VMX
600 * transient structure.
601 *
602 * @returns VBox status code.
603 * @param pVmxTransient Pointer to the VMX transient structure.
604 */
605DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
606{
607 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
608 {
609 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
610 AssertRCReturn(rc, rc);
611 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
612 }
613 return VINF_SUCCESS;
614}
615
616
617/**
618 * Reads the VM-exit interruption error code from the VMCS into the VMX
619 * transient structure.
620 *
621 * @returns VBox status code.
622 * @param pVmxTransient Pointer to the VMX transient structure.
623 */
624DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
625{
626 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
627 {
628 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
629 AssertRCReturn(rc, rc);
630 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
631 }
632 return VINF_SUCCESS;
633}
634
635
636/**
637 * Reads the VM-exit instruction length field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVCpu Pointer to the VMCPU.
642 * @param pVmxTransient Pointer to the VMX transient structure.
643 */
644DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
645{
646 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
647 {
648 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
649 AssertRCReturn(rc, rc);
650 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
651 }
652 return VINF_SUCCESS;
653}
654
655
656/**
657 * Reads the VM-exit instruction-information field from the VMCS into
658 * the VMX transient structure.
659 *
660 * @returns VBox status code.
661 * @param pVmxTransient Pointer to the VMX transient structure.
662 */
663DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
664{
665 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
666 {
667 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
668 AssertRCReturn(rc, rc);
669 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
670 }
671 return VINF_SUCCESS;
672}
673
674
675/**
676 * Reads the exit qualification from the VMCS into the VMX transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
680 * case).
681 * @param pVmxTransient Pointer to the VMX transient structure.
682 */
683DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
684{
685 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
686 {
687 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
688 AssertRCReturn(rc, rc);
689 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
690 }
691 return VINF_SUCCESS;
692}
693
694
695/**
696 * Reads the IDT-vectoring information field from the VMCS into the VMX
697 * transient structure.
698 *
699 * @returns VBox status code.
700 * @param pVmxTransient Pointer to the VMX transient structure.
701 *
702 * @remarks No-long-jump zone!!!
703 */
704DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
705{
706 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
707 {
708 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
709 AssertRCReturn(rc, rc);
710 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
711 }
712 return VINF_SUCCESS;
713}
714
715
716/**
717 * Reads the IDT-vectoring error code from the VMCS into the VMX
718 * transient structure.
719 *
720 * @returns VBox status code.
721 * @param pVmxTransient Pointer to the VMX transient structure.
722 */
723DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
724{
725 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
726 {
727 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
728 AssertRCReturn(rc, rc);
729 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
730 }
731 return VINF_SUCCESS;
732}
733
734
735/**
736 * Enters VMX root mode operation on the current CPU.
737 *
738 * @returns VBox status code.
739 * @param pVM Pointer to the VM (optional, can be NULL, after
740 * a resume).
741 * @param HCPhysCpuPage Physical address of the VMXON region.
742 * @param pvCpuPage Pointer to the VMXON region.
743 */
744static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
745{
746 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
747 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
748 Assert(pvCpuPage);
749 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
750
751 if (pVM)
752 {
753 /* Write the VMCS revision dword to the VMXON region. */
754 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
755 }
756
757 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
758 RTCCUINTREG uEflags = ASMIntDisableFlags();
759
760 /* Enable the VMX bit in CR4 if necessary. */
761 RTCCUINTREG uCr4 = ASMGetCR4();
762 if (!(uCr4 & X86_CR4_VMXE))
763 ASMSetCR4(uCr4 | X86_CR4_VMXE);
764
765 /* Enter VMX root mode. */
766 int rc = VMXEnable(HCPhysCpuPage);
767 if (RT_FAILURE(rc))
768 ASMSetCR4(uCr4);
769
770 /* Restore interrupts. */
771 ASMSetFlags(uEflags);
772 return rc;
773}
774
775
776/**
777 * Exits VMX root mode operation on the current CPU.
778 *
779 * @returns VBox status code.
780 */
781static int hmR0VmxLeaveRootMode(void)
782{
783 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
784
785 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
786 RTCCUINTREG uEflags = ASMIntDisableFlags();
787
788 /* If we're for some reason not in VMX root mode, then don't leave it. */
789 RTCCUINTREG uHostCR4 = ASMGetCR4();
790
791 int rc;
792 if (uHostCR4 & X86_CR4_VMXE)
793 {
794 /* Exit VMX root mode and clear the VMX bit in CR4. */
795 VMXDisable();
796 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
797 rc = VINF_SUCCESS;
798 }
799 else
800 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
801
802 /* Restore interrupts. */
803 ASMSetFlags(uEflags);
804 return rc;
805}
806
807
808/**
809 * Allocates and maps one physically contiguous page. The allocated page is
810 * zero'd out. (Used by various VT-x structures).
811 *
812 * @returns IPRT status code.
813 * @param pMemObj Pointer to the ring-0 memory object.
814 * @param ppVirt Where to store the virtual address of the
815 * allocation.
816 * @param pPhys Where to store the physical address of the
817 * allocation.
818 */
819DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
820{
821 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
822 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
823 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
824
825 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
826 if (RT_FAILURE(rc))
827 return rc;
828 *ppVirt = RTR0MemObjAddress(*pMemObj);
829 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
830 ASMMemZero32(*ppVirt, PAGE_SIZE);
831 return VINF_SUCCESS;
832}
833
834
835/**
836 * Frees and unmaps an allocated physical page.
837 *
838 * @param pMemObj Pointer to the ring-0 memory object.
839 * @param ppVirt Where to re-initialize the virtual address of
840 * allocation as 0.
841 * @param pHCPhys Where to re-initialize the physical address of the
842 * allocation as 0.
843 */
844DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
845{
846 AssertPtr(pMemObj);
847 AssertPtr(ppVirt);
848 AssertPtr(pHCPhys);
849 if (*pMemObj != NIL_RTR0MEMOBJ)
850 {
851 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
852 AssertRC(rc);
853 *pMemObj = NIL_RTR0MEMOBJ;
854 *ppVirt = 0;
855 *pHCPhys = 0;
856 }
857}
858
859
860/**
861 * Worker function to free VT-x related structures.
862 *
863 * @returns IPRT status code.
864 * @param pVM Pointer to the VM.
865 */
866static void hmR0VmxStructsFree(PVM pVM)
867{
868 for (VMCPUID i = 0; i < pVM->cCpus; i++)
869 {
870 PVMCPU pVCpu = &pVM->aCpus[i];
871 AssertPtr(pVCpu);
872
873 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
874 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
875
876 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
878
879 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
881 }
882
883 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
884#ifdef VBOX_WITH_CRASHDUMP_MAGIC
885 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
886#endif
887}
888
889
890/**
891 * Worker function to allocate VT-x related VM structures.
892 *
893 * @returns IPRT status code.
894 * @param pVM Pointer to the VM.
895 */
896static int hmR0VmxStructsAlloc(PVM pVM)
897{
898 /*
899 * Initialize members up-front so we can cleanup properly on allocation failure.
900 */
901#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
902 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
903 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
904 pVM->hm.s.vmx.HCPhys##a_Name = 0;
905
906#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
907 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
908 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
909 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
910
911#ifdef VBOX_WITH_CRASHDUMP_MAGIC
912 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
913#endif
914 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
915
916 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
917 for (VMCPUID i = 0; i < pVM->cCpus; i++)
918 {
919 PVMCPU pVCpu = &pVM->aCpus[i];
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
925 }
926#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
927#undef VMXLOCAL_INIT_VM_MEMOBJ
928
929 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
930 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
931 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
932 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
933
934 /*
935 * Allocate all the VT-x structures.
936 */
937 int rc = VINF_SUCCESS;
938#ifdef VBOX_WITH_CRASHDUMP_MAGIC
939 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
940 if (RT_FAILURE(rc))
941 goto cleanup;
942 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
943 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
944#endif
945
946 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
947 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
948 {
949 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
950 &pVM->hm.s.vmx.HCPhysApicAccess);
951 if (RT_FAILURE(rc))
952 goto cleanup;
953 }
954
955 /*
956 * Initialize per-VCPU VT-x structures.
957 */
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 AssertPtr(pVCpu);
962
963 /* Allocate the VM control structure (VMCS). */
964 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
965 if (RT_FAILURE(rc))
966 goto cleanup;
967
968 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
969 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
970 {
971 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
972 &pVCpu->hm.s.vmx.HCPhysVirtApic);
973 if (RT_FAILURE(rc))
974 goto cleanup;
975 }
976
977 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
978 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
979 {
980 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
981 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
982 if (RT_FAILURE(rc))
983 goto cleanup;
984 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
985 }
986
987 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
988 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
989 if (RT_FAILURE(rc))
990 goto cleanup;
991
992 /* Allocate the VM-exit MSR-load page for the host MSRs. */
993 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
994 if (RT_FAILURE(rc))
995 goto cleanup;
996 }
997
998 return VINF_SUCCESS;
999
1000cleanup:
1001 hmR0VmxStructsFree(pVM);
1002 return rc;
1003}
1004
1005
1006/**
1007 * Does global VT-x initialization (called during module initialization).
1008 *
1009 * @returns VBox status code.
1010 */
1011VMMR0DECL(int) VMXR0GlobalInit(void)
1012{
1013#ifdef HMVMX_USE_FUNCTION_TABLE
1014 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1015# ifdef VBOX_STRICT
1016 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1017 Assert(g_apfnVMExitHandlers[i]);
1018# endif
1019#endif
1020 return VINF_SUCCESS;
1021}
1022
1023
1024/**
1025 * Does global VT-x termination (called during module termination).
1026 */
1027VMMR0DECL(void) VMXR0GlobalTerm()
1028{
1029 /* Nothing to do currently. */
1030}
1031
1032
1033/**
1034 * Sets up and activates VT-x on the current CPU.
1035 *
1036 * @returns VBox status code.
1037 * @param pCpu Pointer to the global CPU info struct.
1038 * @param pVM Pointer to the VM (can be NULL after a host resume
1039 * operation).
1040 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1041 * fEnabledByHost is true).
1042 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1043 * @a fEnabledByHost is true).
1044 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1045 * enable VT-x on the host.
1046 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1047 */
1048VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1049 void *pvMsrs)
1050{
1051 Assert(pCpu);
1052 Assert(pvMsrs);
1053 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1054
1055 /* Enable VT-x if it's not already enabled by the host. */
1056 if (!fEnabledByHost)
1057 {
1058 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1059 if (RT_FAILURE(rc))
1060 return rc;
1061 }
1062
1063 /*
1064 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1065 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1066 */
1067 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1068 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1069 {
1070 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1071 pCpu->fFlushAsidBeforeUse = false;
1072 }
1073 else
1074 pCpu->fFlushAsidBeforeUse = true;
1075
1076 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1077 ++pCpu->cTlbFlushes;
1078
1079 return VINF_SUCCESS;
1080}
1081
1082
1083/**
1084 * Deactivates VT-x on the current CPU.
1085 *
1086 * @returns VBox status code.
1087 * @param pCpu Pointer to the global CPU info struct.
1088 * @param pvCpuPage Pointer to the VMXON region.
1089 * @param HCPhysCpuPage Physical address of the VMXON region.
1090 *
1091 * @remarks This function should never be called when SUPR0EnableVTx() or
1092 * similar was used to enable VT-x on the host.
1093 */
1094VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1095{
1096 NOREF(pCpu);
1097 NOREF(pvCpuPage);
1098 NOREF(HCPhysCpuPage);
1099
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101 return hmR0VmxLeaveRootMode();
1102}
1103
1104
1105/**
1106 * Sets the permission bits for the specified MSR in the MSR bitmap.
1107 *
1108 * @param pVCpu Pointer to the VMCPU.
1109 * @param uMSR The MSR value.
1110 * @param enmRead Whether reading this MSR causes a VM-exit.
1111 * @param enmWrite Whether writing this MSR causes a VM-exit.
1112 */
1113static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1114{
1115 int32_t iBit;
1116 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1117
1118 /*
1119 * Layout:
1120 * 0x000 - 0x3ff - Low MSR read bits
1121 * 0x400 - 0x7ff - High MSR read bits
1122 * 0x800 - 0xbff - Low MSR write bits
1123 * 0xc00 - 0xfff - High MSR write bits
1124 */
1125 if (uMsr <= 0x00001FFF)
1126 iBit = uMsr;
1127 else if ( uMsr >= 0xC0000000
1128 && uMsr <= 0xC0001FFF)
1129 {
1130 iBit = (uMsr - 0xC0000000);
1131 pbMsrBitmap += 0x400;
1132 }
1133 else
1134 {
1135 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1136 return;
1137 }
1138
1139 Assert(iBit <= 0x1fff);
1140 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1141 ASMBitSet(pbMsrBitmap, iBit);
1142 else
1143 ASMBitClear(pbMsrBitmap, iBit);
1144
1145 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1146 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1147 else
1148 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1149}
1150
1151
1152#ifdef VBOX_STRICT
1153/**
1154 * Gets the permission bits for the specified MSR in the MSR bitmap.
1155 *
1156 * @returns VBox status code.
1157 * @retval VINF_SUCCESS if the specified MSR is found.
1158 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1159 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1160 *
1161 * @param pVCpu Pointer to the VMCPU.
1162 * @param uMsr The MSR.
1163 * @param penmRead Where to store the read permissions.
1164 * @param penmWrite Where to store the write permissions.
1165 */
1166static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1167{
1168 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1169 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1170 int32_t iBit;
1171 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1172
1173 /* See hmR0VmxSetMsrPermission() for the layout. */
1174 if (uMsr <= 0x00001FFF)
1175 iBit = uMsr;
1176 else if ( uMsr >= 0xC0000000
1177 && uMsr <= 0xC0001FFF)
1178 {
1179 iBit = (uMsr - 0xC0000000);
1180 pbMsrBitmap += 0x400;
1181 }
1182 else
1183 {
1184 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1185 return VERR_NOT_SUPPORTED;
1186 }
1187
1188 Assert(iBit <= 0x1fff);
1189 if (ASMBitTest(pbMsrBitmap, iBit))
1190 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1191 else
1192 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1193
1194 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1195 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1196 else
1197 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1198 return VINF_SUCCESS;
1199}
1200#endif /* VBOX_STRICT */
1201
1202
1203/**
1204 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1205 * area.
1206 *
1207 * @returns VBox status code.
1208 * @param pVCpu Pointer to the VMCPU.
1209 * @param cMsrs The number of MSRs.
1210 */
1211DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1212{
1213 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1214 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1215 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1216 {
1217 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1218 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1220 }
1221
1222 /* Update number of guest MSRs to load/store across the world-switch. */
1223 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225
1226 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1227 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1228
1229 /* Update the VCPU's copy of the MSR count. */
1230 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1231
1232 return VINF_SUCCESS;
1233}
1234
1235
1236/**
1237 * Adds a new (or updates the value of an existing) guest/host MSR
1238 * pair to be swapped during the world-switch as part of the
1239 * auto-load/store MSR area in the VMCS.
1240 *
1241 * @returns true if the MSR was added -and- its value was updated, false
1242 * otherwise.
1243 * @param pVCpu Pointer to the VMCPU.
1244 * @param uMsr The MSR.
1245 * @param uGuestMsr Value of the guest MSR.
1246 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1247 * necessary.
1248 */
1249static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1250{
1251 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1252 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1253 uint32_t i;
1254 for (i = 0; i < cMsrs; i++)
1255 {
1256 if (pGuestMsr->u32Msr == uMsr)
1257 break;
1258 pGuestMsr++;
1259 }
1260
1261 bool fAdded = false;
1262 if (i == cMsrs)
1263 {
1264 ++cMsrs;
1265 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1266 AssertRC(rc);
1267
1268 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1269 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1270 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1271
1272 fAdded = true;
1273 }
1274
1275 /* Update the MSR values in the auto-load/store MSR area. */
1276 pGuestMsr->u32Msr = uMsr;
1277 pGuestMsr->u64Value = uGuestMsrValue;
1278
1279 /* Create/update the MSR slot in the host MSR area. */
1280 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1281 pHostMsr += i;
1282 pHostMsr->u32Msr = uMsr;
1283
1284 /*
1285 * Update the host MSR only when requested by the caller AND when we're
1286 * adding it to the auto-load/store area. Otherwise, it would have been
1287 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1288 */
1289 bool fUpdatedMsrValue = false;
1290 if ( fAdded
1291 && fUpdateHostMsr)
1292 {
1293 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1294 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1295 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1296 fUpdatedMsrValue = true;
1297 }
1298
1299 return fUpdatedMsrValue;
1300}
1301
1302
1303/**
1304 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1305 * auto-load/store MSR area in the VMCS.
1306 *
1307 * @returns VBox status code.
1308 * @param pVCpu Pointer to the VMCPU.
1309 * @param uMsr The MSR.
1310 */
1311static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1312{
1313 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315 for (uint32_t i = 0; i < cMsrs; i++)
1316 {
1317 /* Find the MSR. */
1318 if (pGuestMsr->u32Msr == uMsr)
1319 {
1320 /* If it's the last MSR, simply reduce the count. */
1321 if (i == cMsrs - 1)
1322 {
1323 --cMsrs;
1324 break;
1325 }
1326
1327 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1328 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1329 pLastGuestMsr += cMsrs - 1;
1330 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1331 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1332
1333 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1334 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 pLastHostMsr += cMsrs - 1;
1336 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1337 pHostMsr->u64Value = pLastHostMsr->u64Value;
1338 --cMsrs;
1339 break;
1340 }
1341 pGuestMsr++;
1342 }
1343
1344 /* Update the VMCS if the count changed (meaning the MSR was found). */
1345 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1346 {
1347 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1348 AssertRCReturn(rc, rc);
1349
1350 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1351 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1352 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1353
1354 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1355 return VINF_SUCCESS;
1356 }
1357
1358 return VERR_NOT_FOUND;
1359}
1360
1361
1362/**
1363 * Checks if the specified guest MSR is part of the auto-load/store area in
1364 * the VMCS.
1365 *
1366 * @returns true if found, false otherwise.
1367 * @param pVCpu Pointer to the VMCPU.
1368 * @param uMsr The MSR to find.
1369 */
1370static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1371{
1372 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1374
1375 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1376 {
1377 if (pGuestMsr->u32Msr == uMsr)
1378 return true;
1379 }
1380 return false;
1381}
1382
1383
1384/**
1385 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1386 *
1387 * @param pVCpu Pointer to the VMCPU.
1388 *
1389 * @remarks No-long-jump zone!!!
1390 */
1391static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1392{
1393 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1394 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1395 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1396 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1397
1398 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1399 {
1400 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1401
1402 /*
1403 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1404 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1405 */
1406 if (pHostMsr->u32Msr == MSR_K6_EFER)
1407 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1408 else
1409 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1410 }
1411
1412 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1413}
1414
1415
1416#if HC_ARCH_BITS == 64
1417/**
1418 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1419 * perform lazy restoration of the host MSRs while leaving VT-x.
1420 *
1421 * @param pVCpu Pointer to the VMCPU.
1422 *
1423 * @remarks No-long-jump zone!!!
1424 */
1425static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1426{
1427 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1428
1429 /*
1430 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1431 */
1432 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1433 {
1434 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1435 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1436 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1437 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1438 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1439 }
1440}
1441
1442
1443/**
1444 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1445 * lazily while leaving VT-x.
1446 *
1447 * @returns true if it does, false otherwise.
1448 * @param pVCpu Pointer to the VMCPU.
1449 * @param uMsr The MSR to check.
1450 */
1451static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1452{
1453 NOREF(pVCpu);
1454 switch (uMsr)
1455 {
1456 case MSR_K8_LSTAR:
1457 case MSR_K6_STAR:
1458 case MSR_K8_SF_MASK:
1459 case MSR_K8_KERNEL_GS_BASE:
1460 return true;
1461 }
1462 return false;
1463}
1464
1465
1466/**
1467 * Saves a set of guests MSRs back into the guest-CPU context.
1468 *
1469 * @param pVCpu Pointer to the VMCPU.
1470 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1471 * out-of-sync. Make sure to update the required fields
1472 * before using them.
1473 *
1474 * @remarks No-long-jump zone!!!
1475 */
1476static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1477{
1478 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1479 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1480
1481 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1482 {
1483 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1484 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1485 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1486 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1487 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1488 }
1489}
1490
1491
1492/**
1493 * Loads a set of guests MSRs to allow read/passthru to the guest.
1494 *
1495 * The name of this function is slightly confusing. This function does NOT
1496 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1497 * common prefix for functions dealing with "lazy restoration" of the shared
1498 * MSRs.
1499 *
1500 * @param pVCpu Pointer to the VMCPU.
1501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1502 * out-of-sync. Make sure to update the required fields
1503 * before using them.
1504 *
1505 * @remarks No-long-jump zone!!!
1506 */
1507static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1508{
1509 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1510 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1511
1512 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1513 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1514 {
1515#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1516 do { \
1517 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1518 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1519 else \
1520 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1521 } while (0)
1522
1523 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1524 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1525 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1526 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1527#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1528 }
1529 else
1530 {
1531 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1532 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1533 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1534 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1535 }
1536 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1537}
1538
1539
1540/**
1541 * Performs lazy restoration of the set of host MSRs if they were previously
1542 * loaded with guest MSR values.
1543 *
1544 * @param pVCpu Pointer to the VMCPU.
1545 *
1546 * @remarks No-long-jump zone!!!
1547 * @remarks The guest MSRs should have been saved back into the guest-CPU
1548 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1549 */
1550static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1551{
1552 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1553 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1554
1555 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1556 {
1557 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1558 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1559 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1560 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1561 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1562 }
1563 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1564}
1565#endif /* HC_ARCH_BITS == 64 */
1566
1567
1568/**
1569 * Verifies that our cached values of the VMCS controls are all
1570 * consistent with what's actually present in the VMCS.
1571 *
1572 * @returns VBox status code.
1573 * @param pVCpu Pointer to the VMCPU.
1574 */
1575static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1576{
1577 uint32_t u32Val;
1578 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1579 AssertRCReturn(rc, rc);
1580 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1581 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1582
1583 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1584 AssertRCReturn(rc, rc);
1585 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1586 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1587
1588 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1589 AssertRCReturn(rc, rc);
1590 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1591 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1592
1593 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1594 AssertRCReturn(rc, rc);
1595 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1596 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1597
1598 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1599 AssertRCReturn(rc, rc);
1600 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1601 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1602
1603 return VINF_SUCCESS;
1604}
1605
1606
1607#ifdef VBOX_STRICT
1608/**
1609 * Verifies that our cached host EFER value has not changed
1610 * since we cached it.
1611 *
1612 * @param pVCpu Pointer to the VMCPU.
1613 */
1614static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1615{
1616 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1617
1618 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1619 {
1620 uint64_t u64Val;
1621 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1622 AssertRC(rc);
1623
1624 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1625 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1626 }
1627}
1628
1629
1630/**
1631 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1632 * VMCS are correct.
1633 *
1634 * @param pVCpu Pointer to the VMCPU.
1635 */
1636static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1637{
1638 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1639
1640 /* Verify MSR counts in the VMCS are what we think it should be. */
1641 uint32_t cMsrs;
1642 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1643 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1646 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1647
1648 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1649 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1650
1651 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1652 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1653 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1654 {
1655 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1656 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1657 pGuestMsr->u32Msr, cMsrs));
1658
1659 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1660 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1661 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1662
1663 /* Verify that the permissions are as expected in the MSR bitmap. */
1664 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1665 {
1666 VMXMSREXITREAD enmRead;
1667 VMXMSREXITWRITE enmWrite;
1668 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1669 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1670 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1671 {
1672 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1673 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1674 }
1675 else
1676 {
1677 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1678 pGuestMsr->u32Msr, cMsrs));
1679 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1680 pGuestMsr->u32Msr, cMsrs));
1681 }
1682 }
1683 }
1684}
1685#endif /* VBOX_STRICT */
1686
1687
1688/**
1689 * Flushes the TLB using EPT.
1690 *
1691 * @returns VBox status code.
1692 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1693 * enmFlush).
1694 * @param enmFlush Type of flush.
1695 *
1696 * @remarks Caller is responsible for making sure this function is called only
1697 * when NestedPaging is supported and providing @a enmFlush that is
1698 * supported by the CPU.
1699 * @remarks Can be called with interrupts disabled.
1700 */
1701static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1702{
1703 uint64_t au64Descriptor[2];
1704 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1705 au64Descriptor[0] = 0;
1706 else
1707 {
1708 Assert(pVCpu);
1709 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1710 }
1711 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1712
1713 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1714 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1715 rc));
1716 if ( RT_SUCCESS(rc)
1717 && pVCpu)
1718 {
1719 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1720 }
1721}
1722
1723
1724/**
1725 * Flushes the TLB using VPID.
1726 *
1727 * @returns VBox status code.
1728 * @param pVM Pointer to the VM.
1729 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1730 * enmFlush).
1731 * @param enmFlush Type of flush.
1732 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1733 * on @a enmFlush).
1734 *
1735 * @remarks Can be called with interrupts disabled.
1736 */
1737static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1738{
1739 NOREF(pVM);
1740 AssertPtr(pVM);
1741 Assert(pVM->hm.s.vmx.fVpid);
1742
1743 uint64_t au64Descriptor[2];
1744 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1745 {
1746 au64Descriptor[0] = 0;
1747 au64Descriptor[1] = 0;
1748 }
1749 else
1750 {
1751 AssertPtr(pVCpu);
1752 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1753 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1754 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1755 au64Descriptor[1] = GCPtr;
1756 }
1757
1758 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1759 AssertMsg(rc == VINF_SUCCESS,
1760 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1761 if ( RT_SUCCESS(rc)
1762 && pVCpu)
1763 {
1764 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1765 }
1766}
1767
1768
1769/**
1770 * Invalidates a guest page by guest virtual address. Only relevant for
1771 * EPT/VPID, otherwise there is nothing really to invalidate.
1772 *
1773 * @returns VBox status code.
1774 * @param pVM Pointer to the VM.
1775 * @param pVCpu Pointer to the VMCPU.
1776 * @param GCVirt Guest virtual address of the page to invalidate.
1777 */
1778VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1779{
1780 AssertPtr(pVM);
1781 AssertPtr(pVCpu);
1782 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1783
1784 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1785 if (!fFlushPending)
1786 {
1787 /*
1788 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1789 * See @bugref{6043} and @bugref{6177}.
1790 *
1791 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1792 * function maybe called in a loop with individual addresses.
1793 */
1794 if (pVM->hm.s.vmx.fVpid)
1795 {
1796 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1797 {
1798 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1799 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1800 }
1801 else
1802 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1803 }
1804 else if (pVM->hm.s.fNestedPaging)
1805 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1806 }
1807
1808 return VINF_SUCCESS;
1809}
1810
1811
1812/**
1813 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1814 * otherwise there is nothing really to invalidate.
1815 *
1816 * @returns VBox status code.
1817 * @param pVM Pointer to the VM.
1818 * @param pVCpu Pointer to the VMCPU.
1819 * @param GCPhys Guest physical address of the page to invalidate.
1820 */
1821VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1822{
1823 NOREF(pVM); NOREF(GCPhys);
1824 LogFlowFunc(("%RGp\n", GCPhys));
1825
1826 /*
1827 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1828 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1829 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1830 */
1831 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1832 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1833 return VINF_SUCCESS;
1834}
1835
1836
1837/**
1838 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1839 * case where neither EPT nor VPID is supported by the CPU.
1840 *
1841 * @param pVM Pointer to the VM.
1842 * @param pVCpu Pointer to the VMCPU.
1843 * @param pCpu Pointer to the global HM struct.
1844 *
1845 * @remarks Called with interrupts disabled.
1846 */
1847static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1848{
1849 AssertPtr(pVCpu);
1850 AssertPtr(pCpu);
1851 NOREF(pVM);
1852
1853 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1854
1855 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1856#if 0
1857 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1858 pVCpu->hm.s.TlbShootdown.cPages = 0;
1859#endif
1860
1861 Assert(pCpu->idCpu != NIL_RTCPUID);
1862 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1863 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1864 pVCpu->hm.s.fForceTLBFlush = false;
1865 return;
1866}
1867
1868
1869/**
1870 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1871 *
1872 * @param pVM Pointer to the VM.
1873 * @param pVCpu Pointer to the VMCPU.
1874 * @param pCpu Pointer to the global HM CPU struct.
1875 * @remarks All references to "ASID" in this function pertains to "VPID" in
1876 * Intel's nomenclature. The reason is, to avoid confusion in compare
1877 * statements since the host-CPU copies are named "ASID".
1878 *
1879 * @remarks Called with interrupts disabled.
1880 */
1881static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1882{
1883#ifdef VBOX_WITH_STATISTICS
1884 bool fTlbFlushed = false;
1885# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1886# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1887 if (!fTlbFlushed) \
1888 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1889 } while (0)
1890#else
1891# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1892# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1893#endif
1894
1895 AssertPtr(pVM);
1896 AssertPtr(pCpu);
1897 AssertPtr(pVCpu);
1898 Assert(pCpu->idCpu != NIL_RTCPUID);
1899
1900 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1901 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1902 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1903
1904 /*
1905 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1906 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1907 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1908 */
1909 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1910 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1911 {
1912 ++pCpu->uCurrentAsid;
1913 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1914 {
1915 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1916 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1917 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1918 }
1919
1920 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1921 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1922 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1923
1924 /*
1925 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1926 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1927 */
1928 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1929 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1930 HMVMX_SET_TAGGED_TLB_FLUSHED();
1931 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1932 }
1933
1934 /* Check for explicit TLB shootdowns. */
1935 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1936 {
1937 /*
1938 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1939 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1940 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1941 * but not guest-physical mappings.
1942 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1943 */
1944 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1945 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1946 HMVMX_SET_TAGGED_TLB_FLUSHED();
1947 }
1948
1949 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1950 * where it is commented out. Support individual entry flushing
1951 * someday. */
1952#if 0
1953 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1954 {
1955 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1956
1957 /*
1958 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1959 * as supported by the CPU.
1960 */
1961 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1962 {
1963 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1964 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1965 }
1966 else
1967 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1968
1969 HMVMX_SET_TAGGED_TLB_FLUSHED();
1970 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1971 pVCpu->hm.s.TlbShootdown.cPages = 0;
1972 }
1973#endif
1974
1975 pVCpu->hm.s.fForceTLBFlush = false;
1976
1977 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1978
1979 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1980 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1981 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1982 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1983 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1984 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1985 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1986 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1987 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1988
1989 /* Update VMCS with the VPID. */
1990 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1991 AssertRC(rc);
1992
1993#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1994}
1995
1996
1997/**
1998 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1999 *
2000 * @returns VBox status code.
2001 * @param pVM Pointer to the VM.
2002 * @param pVCpu Pointer to the VMCPU.
2003 * @param pCpu Pointer to the global HM CPU struct.
2004 *
2005 * @remarks Called with interrupts disabled.
2006 */
2007static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2008{
2009 AssertPtr(pVM);
2010 AssertPtr(pVCpu);
2011 AssertPtr(pCpu);
2012 Assert(pCpu->idCpu != NIL_RTCPUID);
2013 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2014 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2015
2016 /*
2017 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2018 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2019 */
2020 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2021 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2022 {
2023 pVCpu->hm.s.fForceTLBFlush = true;
2024 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2025 }
2026
2027 /* Check for explicit TLB shootdown flushes. */
2028 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2029 {
2030 pVCpu->hm.s.fForceTLBFlush = true;
2031 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2032 }
2033
2034 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2035 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2036
2037 if (pVCpu->hm.s.fForceTLBFlush)
2038 {
2039 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2040 pVCpu->hm.s.fForceTLBFlush = false;
2041 }
2042 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2043 * where it is commented out. Support individual entry flushing
2044 * someday. */
2045#if 0
2046 else
2047 {
2048 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2049 {
2050 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2051 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2052 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2053 }
2054 else
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2056
2057 pVCpu->hm.s.TlbShootdown.cPages = 0;
2058 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2059 }
2060#endif
2061}
2062
2063
2064/**
2065 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2066 *
2067 * @returns VBox status code.
2068 * @param pVM Pointer to the VM.
2069 * @param pVCpu Pointer to the VMCPU.
2070 * @param pCpu Pointer to the global HM CPU struct.
2071 *
2072 * @remarks Called with interrupts disabled.
2073 */
2074static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2075{
2076 AssertPtr(pVM);
2077 AssertPtr(pVCpu);
2078 AssertPtr(pCpu);
2079 Assert(pCpu->idCpu != NIL_RTCPUID);
2080 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2081 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2082
2083 /*
2084 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2085 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2086 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2087 */
2088 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2089 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2090 {
2091 pVCpu->hm.s.fForceTLBFlush = true;
2092 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2093 }
2094
2095 /* Check for explicit TLB shootdown flushes. */
2096 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2097 {
2098 /*
2099 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2100 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2101 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2102 */
2103 pVCpu->hm.s.fForceTLBFlush = true;
2104 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2105 }
2106
2107 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2108 if (pVCpu->hm.s.fForceTLBFlush)
2109 {
2110 ++pCpu->uCurrentAsid;
2111 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2112 {
2113 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2114 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2115 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2116 }
2117
2118 pVCpu->hm.s.fForceTLBFlush = false;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2121 if (pCpu->fFlushAsidBeforeUse)
2122 {
2123 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2124 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2125 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2126 {
2127 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2128 pCpu->fFlushAsidBeforeUse = false;
2129 }
2130 else
2131 {
2132 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2133 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2134 }
2135 }
2136 }
2137 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2138 * where it is commented out. Support individual entry flushing
2139 * someday. */
2140#if 0
2141 else
2142 {
2143 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2144 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2145 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2146 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2147
2148 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2149 {
2150 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2151 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2152 {
2153 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2154 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2155 }
2156 else
2157 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2158
2159 pVCpu->hm.s.TlbShootdown.cPages = 0;
2160 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2161 }
2162 else
2163 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2164 }
2165#endif
2166
2167 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2168 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2169 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2170 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2171 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2172 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2173 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2174
2175 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2176 AssertRC(rc);
2177}
2178
2179
2180/**
2181 * Flushes the guest TLB entry based on CPU capabilities.
2182 *
2183 * @param pVCpu Pointer to the VMCPU.
2184 * @param pCpu Pointer to the global HM CPU struct.
2185 */
2186DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2187{
2188#ifdef HMVMX_ALWAYS_FLUSH_TLB
2189 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2190#endif
2191 PVM pVM = pVCpu->CTX_SUFF(pVM);
2192 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2193 {
2194 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2195 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2196 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2197 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2198 default:
2199 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2200 break;
2201 }
2202
2203 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2204 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2205
2206 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2207}
2208
2209
2210/**
2211 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2212 * TLB entries from the host TLB before VM-entry.
2213 *
2214 * @returns VBox status code.
2215 * @param pVM Pointer to the VM.
2216 */
2217static int hmR0VmxSetupTaggedTlb(PVM pVM)
2218{
2219 /*
2220 * Determine optimal flush type for Nested Paging.
2221 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2222 * guest execution (see hmR3InitFinalizeR0()).
2223 */
2224 if (pVM->hm.s.fNestedPaging)
2225 {
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2229 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2230 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2231 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2232 else
2233 {
2234 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2235 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2236 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237 }
2238
2239 /* Make sure the write-back cacheable memory type for EPT is supported. */
2240 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2241 {
2242 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2243 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245 }
2246 }
2247 else
2248 {
2249 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2250 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2251 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2252 }
2253 }
2254
2255 /*
2256 * Determine optimal flush type for VPID.
2257 */
2258 if (pVM->hm.s.vmx.fVpid)
2259 {
2260 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2261 {
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2263 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2264 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2265 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2266 else
2267 {
2268 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2269 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2270 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2271 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2272 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2274 pVM->hm.s.vmx.fVpid = false;
2275 }
2276 }
2277 else
2278 {
2279 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2280 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282 pVM->hm.s.vmx.fVpid = false;
2283 }
2284 }
2285
2286 /*
2287 * Setup the handler for flushing tagged-TLBs.
2288 */
2289 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2290 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2291 else if (pVM->hm.s.fNestedPaging)
2292 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2293 else if (pVM->hm.s.vmx.fVpid)
2294 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2295 else
2296 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2297 return VINF_SUCCESS;
2298}
2299
2300
2301/**
2302 * Sets up pin-based VM-execution controls in the VMCS.
2303 *
2304 * @returns VBox status code.
2305 * @param pVM Pointer to the VM.
2306 * @param pVCpu Pointer to the VMCPU.
2307 */
2308static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2309{
2310 AssertPtr(pVM);
2311 AssertPtr(pVCpu);
2312
2313 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2314 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2315
2316 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2317 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2318 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2319
2320 /* Enable the VMX preemption timer. */
2321 if (pVM->hm.s.vmx.fUsePreemptTimer)
2322 {
2323 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2325 }
2326
2327 if ((val & zap) != val)
2328 {
2329 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2330 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2331 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2332 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2333 }
2334
2335 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2336 AssertRCReturn(rc, rc);
2337
2338 /* Update VCPU with the currently set pin-based VM-execution controls. */
2339 pVCpu->hm.s.vmx.u32PinCtls = val;
2340 return rc;
2341}
2342
2343
2344/**
2345 * Sets up processor-based VM-execution controls in the VMCS.
2346 *
2347 * @returns VBox status code.
2348 * @param pVM Pointer to the VM.
2349 * @param pVMCPU Pointer to the VMCPU.
2350 */
2351static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2352{
2353 AssertPtr(pVM);
2354 AssertPtr(pVCpu);
2355
2356 int rc = VERR_INTERNAL_ERROR_5;
2357 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2358 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2359
2360 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2361 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2362 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2363 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2364 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2365 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2367
2368 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2369 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2370 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2371 {
2372 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2373 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2374 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2375 }
2376
2377 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2378 if (!pVM->hm.s.fNestedPaging)
2379 {
2380 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2381 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2382 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2383 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2384 }
2385
2386 /* Use TPR shadowing if supported by the CPU. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2388 {
2389 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2390 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2391 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2392 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2393 AssertRCReturn(rc, rc);
2394
2395 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2396 /* CR8 writes causes a VM-exit based on TPR threshold. */
2397 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2398 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2399 }
2400 else
2401 {
2402 /*
2403 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2404 * Set this control only for 64-bit guests.
2405 */
2406 if (pVM->hm.s.fAllow64BitGuests)
2407 {
2408 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2409 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2410 }
2411 }
2412
2413 /* Use MSR-bitmaps if supported by the CPU. */
2414 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2415 {
2416 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2417
2418 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2419 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2420 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2421 AssertRCReturn(rc, rc);
2422
2423 /*
2424 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2425 * automatically as dedicated fields in the VMCS.
2426 */
2427 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2432
2433#if HC_ARCH_BITS == 64
2434 /*
2435 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2436 */
2437 if (pVM->hm.s.fAllow64BitGuests)
2438 {
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 }
2444#endif
2445 }
2446
2447 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2448 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2449 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2450
2451 if ((val & zap) != val)
2452 {
2453 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2454 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2455 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2456 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2457 }
2458
2459 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2460 AssertRCReturn(rc, rc);
2461
2462 /* Update VCPU with the currently set processor-based VM-execution controls. */
2463 pVCpu->hm.s.vmx.u32ProcCtls = val;
2464
2465 /*
2466 * Secondary processor-based VM-execution controls.
2467 */
2468 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2469 {
2470 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2471 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2472
2473 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2474 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2475
2476 if (pVM->hm.s.fNestedPaging)
2477 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2478 else
2479 {
2480 /*
2481 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2482 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2483 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2484 */
2485 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2487 }
2488
2489 if (pVM->hm.s.vmx.fVpid)
2490 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2491
2492 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2493 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2494
2495 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2496 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2497 * done dynamically. */
2498 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2499 {
2500 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2501 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2502 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2503 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2504 AssertRCReturn(rc, rc);
2505 }
2506
2507 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2508 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2509
2510 if ((val & zap) != val)
2511 {
2512 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2513 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2514 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2515 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2516 }
2517
2518 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2519 AssertRCReturn(rc, rc);
2520
2521 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2522 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2523 }
2524 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2525 {
2526 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2527 "available\n"));
2528 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2529 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2530 }
2531
2532 return VINF_SUCCESS;
2533}
2534
2535
2536/**
2537 * Sets up miscellaneous (everything other than Pin & Processor-based
2538 * VM-execution) control fields in the VMCS.
2539 *
2540 * @returns VBox status code.
2541 * @param pVM Pointer to the VM.
2542 * @param pVCpu Pointer to the VMCPU.
2543 */
2544static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2545{
2546 NOREF(pVM);
2547 AssertPtr(pVM);
2548 AssertPtr(pVCpu);
2549
2550 int rc = VERR_GENERAL_FAILURE;
2551
2552 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2553#if 0
2554 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2555 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2556 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2557
2558 /*
2559 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2560 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2561 * We thus use the exception bitmap to control it rather than use both.
2562 */
2563 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2564 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2565
2566 /** @todo Explore possibility of using IO-bitmaps. */
2567 /* All IO & IOIO instructions cause VM-exits. */
2568 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2570
2571 /* Initialize the MSR-bitmap area. */
2572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2575#endif
2576
2577 /* Setup MSR auto-load/store area. */
2578 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2579 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2580 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2581 AssertRCReturn(rc, rc);
2582 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2583 AssertRCReturn(rc, rc);
2584
2585 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2586 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2587 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2588 AssertRCReturn(rc, rc);
2589
2590 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2591 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2592 AssertRCReturn(rc, rc);
2593
2594 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2595#if 0
2596 /* Setup debug controls */
2597 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2598 AssertRCReturn(rc, rc);
2599 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2600 AssertRCReturn(rc, rc);
2601#endif
2602
2603 return rc;
2604}
2605
2606
2607/**
2608 * Sets up the initial exception bitmap in the VMCS based on static conditions
2609 * (i.e. conditions that cannot ever change after starting the VM).
2610 *
2611 * @returns VBox status code.
2612 * @param pVM Pointer to the VM.
2613 * @param pVCpu Pointer to the VMCPU.
2614 */
2615static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2616{
2617 AssertPtr(pVM);
2618 AssertPtr(pVCpu);
2619
2620 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2621
2622 uint32_t u32XcptBitmap = 0;
2623
2624 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2625 if (!pVM->hm.s.fNestedPaging)
2626 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2627
2628 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2629 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2630 AssertRCReturn(rc, rc);
2631 return rc;
2632}
2633
2634
2635/**
2636 * Sets up the initial guest-state mask. The guest-state mask is consulted
2637 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2638 * for the nested virtualization case (as it would cause a VM-exit).
2639 *
2640 * @param pVCpu Pointer to the VMCPU.
2641 */
2642static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2643{
2644 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2645 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2646 return VINF_SUCCESS;
2647}
2648
2649
2650/**
2651 * Does per-VM VT-x initialization.
2652 *
2653 * @returns VBox status code.
2654 * @param pVM Pointer to the VM.
2655 */
2656VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2657{
2658 LogFlowFunc(("pVM=%p\n", pVM));
2659
2660 int rc = hmR0VmxStructsAlloc(pVM);
2661 if (RT_FAILURE(rc))
2662 {
2663 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2664 return rc;
2665 }
2666
2667 return VINF_SUCCESS;
2668}
2669
2670
2671/**
2672 * Does per-VM VT-x termination.
2673 *
2674 * @returns VBox status code.
2675 * @param pVM Pointer to the VM.
2676 */
2677VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2678{
2679 LogFlowFunc(("pVM=%p\n", pVM));
2680
2681#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2682 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2683 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2684#endif
2685 hmR0VmxStructsFree(pVM);
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Sets up the VM for execution under VT-x.
2692 * This function is only called once per-VM during initialization.
2693 *
2694 * @returns VBox status code.
2695 * @param pVM Pointer to the VM.
2696 */
2697VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2698{
2699 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2700 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2701
2702 LogFlowFunc(("pVM=%p\n", pVM));
2703
2704 /*
2705 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2706 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2707 */
2708 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2709 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2710 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2711 || !pVM->hm.s.vmx.pRealModeTSS))
2712 {
2713 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2714 return VERR_INTERNAL_ERROR;
2715 }
2716
2717#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2718 /*
2719 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2720 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2721 */
2722 if ( pVM->hm.s.fAllow64BitGuests
2723 && !HMVMX_IS_64BIT_HOST_MODE())
2724 {
2725 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2726 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2727 }
2728#endif
2729
2730 /* Initialize these always, see hmR3InitFinalizeR0().*/
2731 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2732 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2733
2734 /* Setup the tagged-TLB flush handlers. */
2735 int rc = hmR0VmxSetupTaggedTlb(pVM);
2736 if (RT_FAILURE(rc))
2737 {
2738 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2739 return rc;
2740 }
2741
2742 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2743 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2744#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2745 if ( HMVMX_IS_64BIT_HOST_MODE()
2746 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2747 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2748 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2749 {
2750 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2751 }
2752#endif
2753
2754 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2755 {
2756 PVMCPU pVCpu = &pVM->aCpus[i];
2757 AssertPtr(pVCpu);
2758 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2759
2760 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2761 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2762
2763 /* Set revision dword at the beginning of the VMCS structure. */
2764 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2765
2766 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2767 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2768 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2769 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2770
2771 /* Load this VMCS as the current VMCS. */
2772 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2773 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2774 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2775
2776 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2777 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2778 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2779
2780 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2781 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2782 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2783
2784 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2786 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2787
2788 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2790 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2791
2792 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2794 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2795
2796#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2797 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2799 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2800#endif
2801
2802 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2803 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2805 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2806
2807 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2808
2809 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2810 }
2811
2812 return VINF_SUCCESS;
2813}
2814
2815
2816/**
2817 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2818 * the VMCS.
2819 *
2820 * @returns VBox status code.
2821 * @param pVM Pointer to the VM.
2822 * @param pVCpu Pointer to the VMCPU.
2823 */
2824DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2825{
2826 NOREF(pVM); NOREF(pVCpu);
2827
2828 RTCCUINTREG uReg = ASMGetCR0();
2829 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2830 AssertRCReturn(rc, rc);
2831
2832#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2833 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2834 if (HMVMX_IS_64BIT_HOST_MODE())
2835 {
2836 uint64_t uRegCR3 = HMR0Get64bitCR3();
2837 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2838 }
2839 else
2840#endif
2841 {
2842 uReg = ASMGetCR3();
2843 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2844 }
2845 AssertRCReturn(rc, rc);
2846
2847 uReg = ASMGetCR4();
2848 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2849 AssertRCReturn(rc, rc);
2850 return rc;
2851}
2852
2853
2854#if HC_ARCH_BITS == 64
2855/**
2856 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2857 * requirements. See hmR0VmxSaveHostSegmentRegs().
2858 */
2859# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2860 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2861 { \
2862 bool fValidSelector = true; \
2863 if ((selValue) & X86_SEL_LDT) \
2864 { \
2865 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2866 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2867 } \
2868 if (fValidSelector) \
2869 { \
2870 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2871 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2872 } \
2873 (selValue) = 0; \
2874 }
2875#endif
2876
2877
2878/**
2879 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2880 * the host-state area in the VMCS.
2881 *
2882 * @returns VBox status code.
2883 * @param pVM Pointer to the VM.
2884 * @param pVCpu Pointer to the VMCPU.
2885 */
2886DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2887{
2888 NOREF(pVM);
2889 int rc = VERR_INTERNAL_ERROR_5;
2890
2891#if HC_ARCH_BITS == 64
2892 /*
2893 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2894 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2895 */
2896 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2897 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2898#endif
2899
2900 /*
2901 * Host DS, ES, FS and GS segment registers.
2902 */
2903#if HC_ARCH_BITS == 64
2904 RTSEL uSelDS = ASMGetDS();
2905 RTSEL uSelES = ASMGetES();
2906 RTSEL uSelFS = ASMGetFS();
2907 RTSEL uSelGS = ASMGetGS();
2908#else
2909 RTSEL uSelDS = 0;
2910 RTSEL uSelES = 0;
2911 RTSEL uSelFS = 0;
2912 RTSEL uSelGS = 0;
2913#endif
2914
2915 /* Recalculate which host-state bits need to be manually restored. */
2916 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2917
2918 /*
2919 * Host CS and SS segment registers.
2920 */
2921#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2922 RTSEL uSelCS;
2923 RTSEL uSelSS;
2924 if (HMVMX_IS_64BIT_HOST_MODE())
2925 {
2926 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2927 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2928 }
2929 else
2930 {
2931 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2932 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2933 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2934 }
2935#else
2936 RTSEL uSelCS = ASMGetCS();
2937 RTSEL uSelSS = ASMGetSS();
2938#endif
2939
2940 /*
2941 * Host TR segment register.
2942 */
2943 RTSEL uSelTR = ASMGetTR();
2944
2945#if HC_ARCH_BITS == 64
2946 /*
2947 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2948 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2949 */
2950 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2951 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2952 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2953 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2954# undef VMXLOCAL_ADJUST_HOST_SEG
2955#endif
2956
2957 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2958 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2959 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2960 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2961 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2962 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2963 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2964 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2965 Assert(uSelCS);
2966 Assert(uSelTR);
2967
2968 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2969#if 0
2970 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2971 Assert(uSelSS != 0);
2972#endif
2973
2974 /* Write these host selector fields into the host-state area in the VMCS. */
2975 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2976 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2977#if HC_ARCH_BITS == 64
2978 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2979 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2980 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2982#endif
2983 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2984
2985 /*
2986 * Host GDTR and IDTR.
2987 */
2988 RTGDTR Gdtr;
2989 RT_ZERO(Gdtr);
2990#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2991 if (HMVMX_IS_64BIT_HOST_MODE())
2992 {
2993 X86XDTR64 Gdtr64;
2994 X86XDTR64 Idtr64;
2995 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2996 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2998
2999 Gdtr.cbGdt = Gdtr64.cb;
3000 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3001 }
3002 else
3003#endif
3004 {
3005 RTIDTR Idtr;
3006 ASMGetGDTR(&Gdtr);
3007 ASMGetIDTR(&Idtr);
3008 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3009 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3010
3011#if HC_ARCH_BITS == 64
3012 /*
3013 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3014 * maximum limit (0xffff) on every VM-exit.
3015 */
3016 if (Gdtr.cbGdt != 0xffff)
3017 {
3018 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3019 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3020 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3021 }
3022
3023 /*
3024 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3025 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3026 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3027 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3028 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3029 * hosts where we are pretty sure it won't cause trouble.
3030 */
3031# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3032 if (Idtr.cbIdt < 0x0fff)
3033# else
3034 if (Idtr.cbIdt != 0xffff)
3035# endif
3036 {
3037 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3038 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3039 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3040 }
3041#endif
3042 }
3043
3044 /*
3045 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3046 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3047 */
3048 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3049 {
3050 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3051 return VERR_VMX_INVALID_HOST_STATE;
3052 }
3053
3054 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3055#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3056 if (HMVMX_IS_64BIT_HOST_MODE())
3057 {
3058 /* We need the 64-bit TR base for hybrid darwin. */
3059 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3060 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3061 }
3062 else
3063#endif
3064 {
3065 uintptr_t uTRBase;
3066#if HC_ARCH_BITS == 64
3067 uTRBase = X86DESC64_BASE(pDesc);
3068
3069 /*
3070 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3071 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3072 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3073 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3074 *
3075 * [1] See Intel spec. 3.5 "System Descriptor Types".
3076 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3077 */
3078 Assert(pDesc->System.u4Type == 11);
3079 if ( pDesc->System.u16LimitLow != 0x67
3080 || pDesc->System.u4LimitHigh)
3081 {
3082 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3083 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3084
3085 /* Store the GDTR here as we need it while restoring TR. */
3086 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3087 }
3088#else
3089 uTRBase = X86DESC_BASE(pDesc);
3090#endif
3091 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3092 }
3093 AssertRCReturn(rc, rc);
3094
3095 /*
3096 * Host FS base and GS base.
3097 */
3098#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3099 if (HMVMX_IS_64BIT_HOST_MODE())
3100 {
3101 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3102 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3103 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3104 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3105
3106# if HC_ARCH_BITS == 64
3107 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3108 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3109 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3110 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3111 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3112# endif
3113 }
3114#endif
3115 return rc;
3116}
3117
3118
3119/**
3120 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3121 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3122 * the host after every successful VM-exit.
3123 *
3124 * @returns VBox status code.
3125 * @param pVM Pointer to the VM.
3126 * @param pVCpu Pointer to the VMCPU.
3127 *
3128 * @remarks No-long-jump zone!!!
3129 */
3130DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3131{
3132 NOREF(pVM);
3133
3134 AssertPtr(pVCpu);
3135 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3136
3137 int rc = VINF_SUCCESS;
3138#if HC_ARCH_BITS == 64
3139 if (pVM->hm.s.fAllow64BitGuests)
3140 hmR0VmxLazySaveHostMsrs(pVCpu);
3141#endif
3142
3143 /*
3144 * Host Sysenter MSRs.
3145 */
3146 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3147 AssertRCReturn(rc, rc);
3148#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3149 if (HMVMX_IS_64BIT_HOST_MODE())
3150 {
3151 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3152 AssertRCReturn(rc, rc);
3153 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3154 }
3155 else
3156 {
3157 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3158 AssertRCReturn(rc, rc);
3159 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3160 }
3161#elif HC_ARCH_BITS == 32
3162 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3163 AssertRCReturn(rc, rc);
3164 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3165#else
3166 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3167 AssertRCReturn(rc, rc);
3168 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3169#endif
3170 AssertRCReturn(rc, rc);
3171
3172 /*
3173 * Host EFER MSR.
3174 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3175 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3176 */
3177 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3178 {
3179 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3180 AssertRCReturn(rc, rc);
3181 }
3182
3183 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3184 * hmR0VmxLoadGuestExitCtls() !! */
3185
3186 return rc;
3187}
3188
3189
3190/**
3191 * Figures out if we need to swap the EFER MSR which is
3192 * particularly expensive.
3193 *
3194 * We check all relevant bits. For now, that's everything
3195 * besides LMA/LME, as these two bits are handled by VM-entry,
3196 * see hmR0VmxLoadGuestExitCtls() and
3197 * hmR0VMxLoadGuestEntryCtls().
3198 *
3199 * @returns true if we need to load guest EFER, false otherwise.
3200 * @param pVCpu Pointer to the VMCPU.
3201 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3202 * out-of-sync. Make sure to update the required fields
3203 * before using them.
3204 *
3205 * @remarks Requires EFER, CR4.
3206 * @remarks No-long-jump zone!!!
3207 */
3208static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3209{
3210#ifdef HMVMX_ALWAYS_SWAP_EFER
3211 return true;
3212#endif
3213
3214#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3215 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3216 if (CPUMIsGuestInLongMode(pVCpu))
3217 return false;
3218#endif
3219
3220 PVM pVM = pVCpu->CTX_SUFF(pVM);
3221 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3222 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3223
3224 /*
3225 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3226 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3227 */
3228 if ( CPUMIsGuestInLongMode(pVCpu)
3229 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3230 {
3231 return true;
3232 }
3233
3234 /*
3235 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3236 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3237 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3238 */
3239 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3240 && (pMixedCtx->cr0 & X86_CR0_PG)
3241 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3242 {
3243 /* Assert that host is PAE capable. */
3244 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3245 return true;
3246 }
3247
3248 /** @todo Check the latest Intel spec. for any other bits,
3249 * like SMEP/SMAP? */
3250 return false;
3251}
3252
3253
3254/**
3255 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3256 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3257 * controls".
3258 *
3259 * @returns VBox status code.
3260 * @param pVCpu Pointer to the VMCPU.
3261 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3262 * out-of-sync. Make sure to update the required fields
3263 * before using them.
3264 *
3265 * @remarks Requires EFER.
3266 * @remarks No-long-jump zone!!!
3267 */
3268DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3269{
3270 int rc = VINF_SUCCESS;
3271 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3272 {
3273 PVM pVM = pVCpu->CTX_SUFF(pVM);
3274 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3275 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3276
3277 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3278 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3279
3280 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3281 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3282 {
3283 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3284 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3285 }
3286 else
3287 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3288
3289 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3290 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3291 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3292 {
3293 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3294 Log4(("Load: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3295 }
3296
3297 /*
3298 * The following should -not- be set (since we're not in SMM mode):
3299 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3300 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3301 */
3302
3303 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3304 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3305
3306 if ((val & zap) != val)
3307 {
3308 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3309 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3310 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3311 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3312 }
3313
3314 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3315 AssertRCReturn(rc, rc);
3316
3317 /* Update VCPU with the currently set VM-exit controls. */
3318 pVCpu->hm.s.vmx.u32EntryCtls = val;
3319 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3320 }
3321 return rc;
3322}
3323
3324
3325/**
3326 * Sets up the VM-exit controls in the VMCS.
3327 *
3328 * @returns VBox status code.
3329 * @param pVM Pointer to the VM.
3330 * @param pVCpu Pointer to the VMCPU.
3331 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3332 * out-of-sync. Make sure to update the required fields
3333 * before using them.
3334 *
3335 * @remarks Requires EFER.
3336 */
3337DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3338{
3339 NOREF(pMixedCtx);
3340
3341 int rc = VINF_SUCCESS;
3342 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3343 {
3344 PVM pVM = pVCpu->CTX_SUFF(pVM);
3345 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3346 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3347
3348 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3349 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3350
3351 /*
3352 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3353 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3354 */
3355#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3356 if (HMVMX_IS_64BIT_HOST_MODE())
3357 {
3358 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3359 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3360 }
3361 else
3362 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3363#else
3364 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3365 {
3366 /* The switcher returns to long mode, EFER is managed by the switcher. */
3367 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3368 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3369 }
3370 else
3371 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3372#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3373
3374 /* If the newer VMCS fields for managing EFER exists, use it. */
3375 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3376 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3377 {
3378 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3379 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3380 Log4(("Load: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3381 }
3382
3383 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3384 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3385
3386 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3387 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3388 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3389
3390 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3391 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3392
3393 if ((val & zap) != val)
3394 {
3395 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3396 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3397 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3398 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3399 }
3400
3401 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3402 AssertRCReturn(rc, rc);
3403
3404 /* Update VCPU with the currently set VM-exit controls. */
3405 pVCpu->hm.s.vmx.u32ExitCtls = val;
3406 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3407 }
3408 return rc;
3409}
3410
3411
3412/**
3413 * Loads the guest APIC and related state.
3414 *
3415 * @returns VBox status code.
3416 * @param pVM Pointer to the VM.
3417 * @param pVCpu Pointer to the VMCPU.
3418 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3419 * out-of-sync. Make sure to update the required fields
3420 * before using them.
3421 */
3422DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3423{
3424 NOREF(pMixedCtx);
3425
3426 int rc = VINF_SUCCESS;
3427 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3428 {
3429 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3430 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3431 {
3432 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3433
3434 bool fPendingIntr = false;
3435 uint8_t u8Tpr = 0;
3436 uint8_t u8PendingIntr = 0;
3437 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3438 AssertRCReturn(rc, rc);
3439
3440 /*
3441 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3442 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3443 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3444 * the interrupt when we VM-exit for other reasons.
3445 */
3446 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3447 uint32_t u32TprThreshold = 0;
3448 if (fPendingIntr)
3449 {
3450 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3451 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3452 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3453 if (u8PendingPriority <= u8TprPriority)
3454 u32TprThreshold = u8PendingPriority;
3455 else
3456 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3457 }
3458 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3459
3460 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3461 AssertRCReturn(rc, rc);
3462 }
3463
3464 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3465 }
3466 return rc;
3467}
3468
3469
3470/**
3471 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3472 *
3473 * @returns Guest's interruptibility-state.
3474 * @param pVCpu Pointer to the VMCPU.
3475 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3476 * out-of-sync. Make sure to update the required fields
3477 * before using them.
3478 *
3479 * @remarks No-long-jump zone!!!
3480 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3481 */
3482DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3483{
3484 /*
3485 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3486 * inhibit interrupts or clear any existing interrupt-inhibition.
3487 */
3488 uint32_t uIntrState = 0;
3489 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3490 {
3491 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3492 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3493 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3494 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3495 {
3496 /*
3497 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3498 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3499 */
3500 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3501 }
3502 else if (pMixedCtx->eflags.Bits.u1IF)
3503 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3504 else
3505 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3506 }
3507 return uIntrState;
3508}
3509
3510
3511/**
3512 * Loads the guest's interruptibility-state into the guest-state area in the
3513 * VMCS.
3514 *
3515 * @returns VBox status code.
3516 * @param pVCpu Pointer to the VMCPU.
3517 * @param uIntrState The interruptibility-state to set.
3518 */
3519static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3520{
3521 NOREF(pVCpu);
3522 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3523 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3524 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3525 AssertRCReturn(rc, rc);
3526 return rc;
3527}
3528
3529
3530/**
3531 * Loads the guest's RIP into the guest-state area in the VMCS.
3532 *
3533 * @returns VBox status code.
3534 * @param pVCpu Pointer to the VMCPU.
3535 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3536 * out-of-sync. Make sure to update the required fields
3537 * before using them.
3538 *
3539 * @remarks No-long-jump zone!!!
3540 */
3541static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3542{
3543 int rc = VINF_SUCCESS;
3544 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3545 {
3546 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3547 AssertRCReturn(rc, rc);
3548
3549 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3550 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3551 }
3552 return rc;
3553}
3554
3555
3556/**
3557 * Loads the guest's RSP into the guest-state area in the VMCS.
3558 *
3559 * @returns VBox status code.
3560 * @param pVCpu Pointer to the VMCPU.
3561 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3562 * out-of-sync. Make sure to update the required fields
3563 * before using them.
3564 *
3565 * @remarks No-long-jump zone!!!
3566 */
3567static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3568{
3569 int rc = VINF_SUCCESS;
3570 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3571 {
3572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3573 AssertRCReturn(rc, rc);
3574
3575 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3576 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu Pointer to the VMCPU.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3594{
3595 int rc = VINF_SUCCESS;
3596 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3597 {
3598 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3599 Let us assert it as such and use 32-bit VMWRITE. */
3600 Assert(!(pMixedCtx->rflags.u64 >> 32));
3601 X86EFLAGS Eflags = pMixedCtx->eflags;
3602 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3603 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3604
3605 /*
3606 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3607 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3608 */
3609 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3610 {
3611 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3612 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3613 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3614 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3615 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3616 }
3617
3618 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3619 AssertRCReturn(rc, rc);
3620
3621 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3622 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3623 }
3624 return rc;
3625}
3626
3627
3628/**
3629 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3630 *
3631 * @returns VBox status code.
3632 * @param pVCpu Pointer to the VMCPU.
3633 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3634 * out-of-sync. Make sure to update the required fields
3635 * before using them.
3636 *
3637 * @remarks No-long-jump zone!!!
3638 */
3639DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3640{
3641 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3642 AssertRCReturn(rc, rc);
3643 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3644 AssertRCReturn(rc, rc);
3645 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3646 AssertRCReturn(rc, rc);
3647 return rc;
3648}
3649
3650
3651/**
3652 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3653 * CR0 is partially shared with the host and we have to consider the FPU bits.
3654 *
3655 * @returns VBox status code.
3656 * @param pVM Pointer to the VM.
3657 * @param pVCpu Pointer to the VMCPU.
3658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3659 * out-of-sync. Make sure to update the required fields
3660 * before using them.
3661 *
3662 * @remarks No-long-jump zone!!!
3663 */
3664static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3665{
3666 /*
3667 * Guest CR0.
3668 * Guest FPU.
3669 */
3670 int rc = VINF_SUCCESS;
3671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3672 {
3673 Assert(!(pMixedCtx->cr0 >> 32));
3674 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3675 PVM pVM = pVCpu->CTX_SUFF(pVM);
3676
3677 /* The guest's view (read access) of its CR0 is unblemished. */
3678 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3679 AssertRCReturn(rc, rc);
3680 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3681
3682 /* Setup VT-x's view of the guest CR0. */
3683 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3684 if (pVM->hm.s.fNestedPaging)
3685 {
3686 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3687 {
3688 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3689 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3691 }
3692 else
3693 {
3694 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3695 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3696 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3697 }
3698
3699 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3700 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3701 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3702
3703 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3704 AssertRCReturn(rc, rc);
3705 }
3706 else
3707 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3708
3709 /*
3710 * Guest FPU bits.
3711 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3712 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3713 */
3714 u32GuestCR0 |= X86_CR0_NE;
3715 bool fInterceptNM = false;
3716 if (CPUMIsGuestFPUStateActive(pVCpu))
3717 {
3718 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3719 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3720 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3721 }
3722 else
3723 {
3724 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3725 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3726 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3727 }
3728
3729 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3730 bool fInterceptMF = false;
3731 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3732 fInterceptMF = true;
3733
3734 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3735 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3736 {
3737 Assert(PDMVmmDevHeapIsEnabled(pVM));
3738 Assert(pVM->hm.s.vmx.pRealModeTSS);
3739 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3740 fInterceptNM = true;
3741 fInterceptMF = true;
3742 }
3743 else
3744 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3745
3746 if (fInterceptNM)
3747 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3748 else
3749 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3750
3751 if (fInterceptMF)
3752 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3753 else
3754 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3755
3756 /* Additional intercepts for debugging, define these yourself explicitly. */
3757#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3758 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3759 | RT_BIT(X86_XCPT_BP)
3760 | RT_BIT(X86_XCPT_DB)
3761 | RT_BIT(X86_XCPT_DE)
3762 | RT_BIT(X86_XCPT_NM)
3763 | RT_BIT(X86_XCPT_TS)
3764 | RT_BIT(X86_XCPT_UD)
3765 | RT_BIT(X86_XCPT_NP)
3766 | RT_BIT(X86_XCPT_SS)
3767 | RT_BIT(X86_XCPT_GP)
3768 | RT_BIT(X86_XCPT_PF)
3769 | RT_BIT(X86_XCPT_MF)
3770 ;
3771#elif defined(HMVMX_ALWAYS_TRAP_PF)
3772 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3773#endif
3774
3775 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3776
3777 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3778 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3779 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3780 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3781 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3782 else
3783 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3784
3785 u32GuestCR0 |= uSetCR0;
3786 u32GuestCR0 &= uZapCR0;
3787 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3788
3789 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3790 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3791 AssertRCReturn(rc, rc);
3792 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3793 AssertRCReturn(rc, rc);
3794 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3795
3796 /*
3797 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3798 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3799 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3800 */
3801 uint32_t u32CR0Mask = 0;
3802 u32CR0Mask = X86_CR0_PE
3803 | X86_CR0_NE
3804 | X86_CR0_WP
3805 | X86_CR0_PG
3806 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3807 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3808 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3809
3810 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3811 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3812 * and @bugref{6944}. */
3813#if 0
3814 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3815 u32CR0Mask &= ~X86_CR0_PE;
3816#endif
3817 if (pVM->hm.s.fNestedPaging)
3818 u32CR0Mask &= ~X86_CR0_WP;
3819
3820 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3821 if (fInterceptNM)
3822 {
3823 u32CR0Mask |= X86_CR0_TS
3824 | X86_CR0_MP;
3825 }
3826
3827 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3828 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3829 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3830 AssertRCReturn(rc, rc);
3831 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3832
3833 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3834 }
3835 return rc;
3836}
3837
3838
3839/**
3840 * Loads the guest control registers (CR3, CR4) into the guest-state area
3841 * in the VMCS.
3842 *
3843 * @returns VBox status code.
3844 * @param pVM Pointer to the VM.
3845 * @param pVCpu Pointer to the VMCPU.
3846 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3847 * out-of-sync. Make sure to update the required fields
3848 * before using them.
3849 *
3850 * @remarks No-long-jump zone!!!
3851 */
3852static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3853{
3854 int rc = VINF_SUCCESS;
3855 PVM pVM = pVCpu->CTX_SUFF(pVM);
3856
3857 /*
3858 * Guest CR2.
3859 * It's always loaded in the assembler code. Nothing to do here.
3860 */
3861
3862 /*
3863 * Guest CR3.
3864 */
3865 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3866 {
3867 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3868 if (pVM->hm.s.fNestedPaging)
3869 {
3870 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3871
3872 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3873 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3874 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3875 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3876
3877 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3878 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3879 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3880
3881 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3882 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3883 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3884 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3885
3886 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3887 AssertRCReturn(rc, rc);
3888 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3889
3890 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3891 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3892 {
3893 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3894 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3895 {
3896 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3897 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3898 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3899 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3900 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3901 }
3902
3903 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3904 have Unrestricted Execution to handle the guest when it's not using paging. */
3905 GCPhysGuestCR3 = pMixedCtx->cr3;
3906 }
3907 else
3908 {
3909 /*
3910 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3911 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3912 * EPT takes care of translating it to host-physical addresses.
3913 */
3914 RTGCPHYS GCPhys;
3915 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3916 Assert(PDMVmmDevHeapIsEnabled(pVM));
3917
3918 /* We obtain it here every time as the guest could have relocated this PCI region. */
3919 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3920 AssertRCReturn(rc, rc);
3921
3922 GCPhysGuestCR3 = GCPhys;
3923 }
3924
3925 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3926 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3927 }
3928 else
3929 {
3930 /* Non-nested paging case, just use the hypervisor's CR3. */
3931 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3932
3933 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3934 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3935 }
3936 AssertRCReturn(rc, rc);
3937
3938 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3939 }
3940
3941 /*
3942 * Guest CR4.
3943 */
3944 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3945 {
3946 Assert(!(pMixedCtx->cr4 >> 32));
3947 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3948
3949 /* The guest's view of its CR4 is unblemished. */
3950 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3951 AssertRCReturn(rc, rc);
3952 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3953
3954 /* Setup VT-x's view of the guest CR4. */
3955 /*
3956 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3957 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3958 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3959 */
3960 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3961 {
3962 Assert(pVM->hm.s.vmx.pRealModeTSS);
3963 Assert(PDMVmmDevHeapIsEnabled(pVM));
3964 u32GuestCR4 &= ~X86_CR4_VME;
3965 }
3966
3967 if (pVM->hm.s.fNestedPaging)
3968 {
3969 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3970 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3971 {
3972 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3973 u32GuestCR4 |= X86_CR4_PSE;
3974 /* Our identity mapping is a 32-bit page directory. */
3975 u32GuestCR4 &= ~X86_CR4_PAE;
3976 }
3977 /* else use guest CR4.*/
3978 }
3979 else
3980 {
3981 /*
3982 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3983 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3984 */
3985 switch (pVCpu->hm.s.enmShadowMode)
3986 {
3987 case PGMMODE_REAL: /* Real-mode. */
3988 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3989 case PGMMODE_32_BIT: /* 32-bit paging. */
3990 {
3991 u32GuestCR4 &= ~X86_CR4_PAE;
3992 break;
3993 }
3994
3995 case PGMMODE_PAE: /* PAE paging. */
3996 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3997 {
3998 u32GuestCR4 |= X86_CR4_PAE;
3999 break;
4000 }
4001
4002 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4003 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4004#ifdef VBOX_ENABLE_64_BITS_GUESTS
4005 break;
4006#endif
4007 default:
4008 AssertFailed();
4009 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4010 }
4011 }
4012
4013 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4014 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4016 u32GuestCR4 |= uSetCR4;
4017 u32GuestCR4 &= uZapCR4;
4018
4019 /* Write VT-x's view of the guest CR4 into the VMCS. */
4020 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
4021 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4022 AssertRCReturn(rc, rc);
4023
4024 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4025 uint32_t u32CR4Mask = 0;
4026 u32CR4Mask = X86_CR4_VME
4027 | X86_CR4_PAE
4028 | X86_CR4_PGE
4029 | X86_CR4_PSE
4030 | X86_CR4_VMXE;
4031 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4032 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4033 AssertRCReturn(rc, rc);
4034
4035 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4036 }
4037 return rc;
4038}
4039
4040
4041/**
4042 * Loads the guest debug registers into the guest-state area in the VMCS.
4043 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4044 *
4045 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4046 *
4047 * @returns VBox status code.
4048 * @param pVCpu Pointer to the VMCPU.
4049 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4050 * out-of-sync. Make sure to update the required fields
4051 * before using them.
4052 *
4053 * @remarks No-long-jump zone!!!
4054 */
4055static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4056{
4057 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4058 return VINF_SUCCESS;
4059
4060#ifdef VBOX_STRICT
4061 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4062 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4063 {
4064 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4065 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4066 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4067 }
4068#endif
4069
4070 int rc;
4071 PVM pVM = pVCpu->CTX_SUFF(pVM);
4072 bool fInterceptDB = false;
4073 bool fInterceptMovDRx = false;
4074 if ( pVCpu->hm.s.fSingleInstruction
4075 || DBGFIsStepping(pVCpu))
4076 {
4077 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4078 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4079 {
4080 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4081 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4082 AssertRCReturn(rc, rc);
4083 Assert(fInterceptDB == false);
4084 }
4085 else
4086 {
4087 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4088 pVCpu->hm.s.fClearTrapFlag = true;
4089 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4090 fInterceptDB = true;
4091 }
4092 }
4093
4094 if ( fInterceptDB
4095 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4096 {
4097 /*
4098 * Use the combined guest and host DRx values found in the hypervisor
4099 * register set because the debugger has breakpoints active or someone
4100 * is single stepping on the host side without a monitor trap flag.
4101 *
4102 * Note! DBGF expects a clean DR6 state before executing guest code.
4103 */
4104#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4105 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4106 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4107 {
4108 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4109 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4110 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4111 }
4112 else
4113#endif
4114 if (!CPUMIsHyperDebugStateActive(pVCpu))
4115 {
4116 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4117 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4118 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4119 }
4120
4121 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4122 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4123 AssertRCReturn(rc, rc);
4124
4125 pVCpu->hm.s.fUsingHyperDR7 = true;
4126 fInterceptDB = true;
4127 fInterceptMovDRx = true;
4128 }
4129 else
4130 {
4131 /*
4132 * If the guest has enabled debug registers, we need to load them prior to
4133 * executing guest code so they'll trigger at the right time.
4134 */
4135 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4136 {
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4138 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4139 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4144 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4145 }
4146 else
4147#endif
4148 if (!CPUMIsGuestDebugStateActive(pVCpu))
4149 {
4150 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4151 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4152 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4153 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4154 }
4155 Assert(!fInterceptDB);
4156 Assert(!fInterceptMovDRx);
4157 }
4158 /*
4159 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160 * must intercept #DB in order to maintain a correct DR6 guest value.
4161 */
4162#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4163 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4164 && !CPUMIsGuestDebugStateActive(pVCpu))
4165#else
4166 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4167#endif
4168 {
4169 fInterceptMovDRx = true;
4170 fInterceptDB = true;
4171 }
4172
4173 /* Update guest DR7. */
4174 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4175 AssertRCReturn(rc, rc);
4176
4177 pVCpu->hm.s.fUsingHyperDR7 = false;
4178 }
4179
4180 /*
4181 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4182 */
4183 if (fInterceptDB)
4184 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4185 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4186 {
4187#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4188 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4189#endif
4190 }
4191 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4192 AssertRCReturn(rc, rc);
4193
4194 /*
4195 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4196 */
4197 if (fInterceptMovDRx)
4198 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4199 else
4200 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4201 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4202 AssertRCReturn(rc, rc);
4203
4204 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4205 return VINF_SUCCESS;
4206}
4207
4208
4209#ifdef VBOX_STRICT
4210/**
4211 * Strict function to validate segment registers.
4212 *
4213 * @remarks ASSUMES CR0 is up to date.
4214 */
4215static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4216{
4217 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4218 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4219 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4220 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4221 && ( !CPUMIsGuestInRealModeEx(pCtx)
4222 && !CPUMIsGuestInV86ModeEx(pCtx)))
4223 {
4224 /* Protected mode checks */
4225 /* CS */
4226 Assert(pCtx->cs.Attr.n.u1Present);
4227 Assert(!(pCtx->cs.Attr.u & 0xf00));
4228 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4229 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4230 || !(pCtx->cs.Attr.n.u1Granularity));
4231 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4232 || (pCtx->cs.Attr.n.u1Granularity));
4233 /* CS cannot be loaded with NULL in protected mode. */
4234 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4235 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4236 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4237 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4238 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4239 else
4240 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4241 /* SS */
4242 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4243 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4244 if ( !(pCtx->cr0 & X86_CR0_PE)
4245 || pCtx->cs.Attr.n.u4Type == 3)
4246 {
4247 Assert(!pCtx->ss.Attr.n.u2Dpl);
4248 }
4249 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4250 {
4251 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4252 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4253 Assert(pCtx->ss.Attr.n.u1Present);
4254 Assert(!(pCtx->ss.Attr.u & 0xf00));
4255 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4256 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4257 || !(pCtx->ss.Attr.n.u1Granularity));
4258 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4259 || (pCtx->ss.Attr.n.u1Granularity));
4260 }
4261 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4262 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4263 {
4264 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4265 Assert(pCtx->ds.Attr.n.u1Present);
4266 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4267 Assert(!(pCtx->ds.Attr.u & 0xf00));
4268 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4269 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4270 || !(pCtx->ds.Attr.n.u1Granularity));
4271 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4272 || (pCtx->ds.Attr.n.u1Granularity));
4273 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4274 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4275 }
4276 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4277 {
4278 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4279 Assert(pCtx->es.Attr.n.u1Present);
4280 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4281 Assert(!(pCtx->es.Attr.u & 0xf00));
4282 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4283 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4284 || !(pCtx->es.Attr.n.u1Granularity));
4285 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4286 || (pCtx->es.Attr.n.u1Granularity));
4287 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4288 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4289 }
4290 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4291 {
4292 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4293 Assert(pCtx->fs.Attr.n.u1Present);
4294 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4295 Assert(!(pCtx->fs.Attr.u & 0xf00));
4296 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4297 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4298 || !(pCtx->fs.Attr.n.u1Granularity));
4299 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4300 || (pCtx->fs.Attr.n.u1Granularity));
4301 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4302 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4303 }
4304 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4305 {
4306 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4307 Assert(pCtx->gs.Attr.n.u1Present);
4308 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4309 Assert(!(pCtx->gs.Attr.u & 0xf00));
4310 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4311 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4312 || !(pCtx->gs.Attr.n.u1Granularity));
4313 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4314 || (pCtx->gs.Attr.n.u1Granularity));
4315 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4316 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4317 }
4318 /* 64-bit capable CPUs. */
4319# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4320 if (HMVMX_IS_64BIT_HOST_MODE())
4321 {
4322 Assert(!(pCtx->cs.u64Base >> 32));
4323 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4324 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4325 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4326 }
4327# endif
4328 }
4329 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4330 || ( CPUMIsGuestInRealModeEx(pCtx)
4331 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4332 {
4333 /* Real and v86 mode checks. */
4334 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4335 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4336 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4337 {
4338 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4339 }
4340 else
4341 {
4342 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4343 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4344 }
4345
4346 /* CS */
4347 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4348 Assert(pCtx->cs.u32Limit == 0xffff);
4349 Assert(u32CSAttr == 0xf3);
4350 /* SS */
4351 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4352 Assert(pCtx->ss.u32Limit == 0xffff);
4353 Assert(u32SSAttr == 0xf3);
4354 /* DS */
4355 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4356 Assert(pCtx->ds.u32Limit == 0xffff);
4357 Assert(u32DSAttr == 0xf3);
4358 /* ES */
4359 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4360 Assert(pCtx->es.u32Limit == 0xffff);
4361 Assert(u32ESAttr == 0xf3);
4362 /* FS */
4363 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4364 Assert(pCtx->fs.u32Limit == 0xffff);
4365 Assert(u32FSAttr == 0xf3);
4366 /* GS */
4367 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4368 Assert(pCtx->gs.u32Limit == 0xffff);
4369 Assert(u32GSAttr == 0xf3);
4370 /* 64-bit capable CPUs. */
4371# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4372 if (HMVMX_IS_64BIT_HOST_MODE())
4373 {
4374 Assert(!(pCtx->cs.u64Base >> 32));
4375 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4376 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4377 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4378 }
4379# endif
4380 }
4381}
4382#endif /* VBOX_STRICT */
4383
4384
4385/**
4386 * Writes a guest segment register into the guest-state area in the VMCS.
4387 *
4388 * @returns VBox status code.
4389 * @param pVCpu Pointer to the VMCPU.
4390 * @param idxSel Index of the selector in the VMCS.
4391 * @param idxLimit Index of the segment limit in the VMCS.
4392 * @param idxBase Index of the segment base in the VMCS.
4393 * @param idxAccess Index of the access rights of the segment in the VMCS.
4394 * @param pSelReg Pointer to the segment selector.
4395 *
4396 * @remarks No-long-jump zone!!!
4397 */
4398static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4399 uint32_t idxAccess, PCPUMSELREG pSelReg)
4400{
4401 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4402 AssertRCReturn(rc, rc);
4403 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4404 AssertRCReturn(rc, rc);
4405 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4406 AssertRCReturn(rc, rc);
4407
4408 uint32_t u32Access = pSelReg->Attr.u;
4409 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4410 {
4411 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4412 u32Access = 0xf3;
4413 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4414 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4415 }
4416 else
4417 {
4418 /*
4419 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4420 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4421 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4422 * loaded in protected-mode have their attribute as 0.
4423 */
4424 if (!u32Access)
4425 u32Access = X86DESCATTR_UNUSABLE;
4426 }
4427
4428 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4429 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4430 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4431
4432 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4433 AssertRCReturn(rc, rc);
4434 return rc;
4435}
4436
4437
4438/**
4439 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4440 * into the guest-state area in the VMCS.
4441 *
4442 * @returns VBox status code.
4443 * @param pVM Pointer to the VM.
4444 * @param pVCPU Pointer to the VMCPU.
4445 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4446 * out-of-sync. Make sure to update the required fields
4447 * before using them.
4448 *
4449 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4450 * @remarks No-long-jump zone!!!
4451 */
4452static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4453{
4454 int rc = VERR_INTERNAL_ERROR_5;
4455 PVM pVM = pVCpu->CTX_SUFF(pVM);
4456
4457 /*
4458 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4459 */
4460 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4461 {
4462 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4463 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4464 {
4465 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4466 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4467 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4468 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4469 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4470 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4471 }
4472
4473#ifdef VBOX_WITH_REM
4474 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4475 {
4476 Assert(pVM->hm.s.vmx.pRealModeTSS);
4477 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4478 if ( pVCpu->hm.s.vmx.fWasInRealMode
4479 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4480 {
4481 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4482 in real-mode (e.g. OpenBSD 4.0) */
4483 REMFlushTBs(pVM);
4484 Log4(("Load: Switch to protected mode detected!\n"));
4485 pVCpu->hm.s.vmx.fWasInRealMode = false;
4486 }
4487 }
4488#endif
4489 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4490 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4491 AssertRCReturn(rc, rc);
4492 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4493 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4494 AssertRCReturn(rc, rc);
4495 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4496 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4497 AssertRCReturn(rc, rc);
4498 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4499 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4500 AssertRCReturn(rc, rc);
4501 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4502 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4503 AssertRCReturn(rc, rc);
4504 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4505 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4506 AssertRCReturn(rc, rc);
4507
4508#ifdef VBOX_STRICT
4509 /* Validate. */
4510 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4511#endif
4512
4513 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4514 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4515 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4516 }
4517
4518 /*
4519 * Guest TR.
4520 */
4521 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4522 {
4523 /*
4524 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4525 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4526 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4527 */
4528 uint16_t u16Sel = 0;
4529 uint32_t u32Limit = 0;
4530 uint64_t u64Base = 0;
4531 uint32_t u32AccessRights = 0;
4532
4533 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4534 {
4535 u16Sel = pMixedCtx->tr.Sel;
4536 u32Limit = pMixedCtx->tr.u32Limit;
4537 u64Base = pMixedCtx->tr.u64Base;
4538 u32AccessRights = pMixedCtx->tr.Attr.u;
4539 }
4540 else
4541 {
4542 Assert(pVM->hm.s.vmx.pRealModeTSS);
4543 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4544
4545 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4546 RTGCPHYS GCPhys;
4547 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4548 AssertRCReturn(rc, rc);
4549
4550 X86DESCATTR DescAttr;
4551 DescAttr.u = 0;
4552 DescAttr.n.u1Present = 1;
4553 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4554
4555 u16Sel = 0;
4556 u32Limit = HM_VTX_TSS_SIZE;
4557 u64Base = GCPhys; /* in real-mode phys = virt. */
4558 u32AccessRights = DescAttr.u;
4559 }
4560
4561 /* Validate. */
4562 Assert(!(u16Sel & RT_BIT(2)));
4563 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4564 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4565 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4566 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4567 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4568 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4569 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4570 Assert( (u32Limit & 0xfff) == 0xfff
4571 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4572 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4573 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4574
4575 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4576 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4577 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4578 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4579
4580 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4581 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4582 }
4583
4584 /*
4585 * Guest GDTR.
4586 */
4587 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4588 {
4589 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4590 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4591
4592 /* Validate. */
4593 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4594
4595 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4596 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4597 }
4598
4599 /*
4600 * Guest LDTR.
4601 */
4602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4603 {
4604 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4605 uint32_t u32Access = 0;
4606 if (!pMixedCtx->ldtr.Attr.u)
4607 u32Access = X86DESCATTR_UNUSABLE;
4608 else
4609 u32Access = pMixedCtx->ldtr.Attr.u;
4610
4611 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4612 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4613 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4615
4616 /* Validate. */
4617 if (!(u32Access & X86DESCATTR_UNUSABLE))
4618 {
4619 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4620 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4621 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4622 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4623 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4624 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4625 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4626 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4627 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4628 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4629 }
4630
4631 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4632 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4633 }
4634
4635 /*
4636 * Guest IDTR.
4637 */
4638 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4639 {
4640 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4641 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4642
4643 /* Validate. */
4644 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4645
4646 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4647 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4648 }
4649
4650 return VINF_SUCCESS;
4651}
4652
4653
4654/**
4655 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4656 * areas. These MSRs will automatically be loaded to the host CPU on every
4657 * successful VM entry and stored from the host CPU on every successful VM-exit.
4658 *
4659 * This also creates/updates MSR slots for the host MSRs. The actual host
4660 * MSR values are -not- updated here for performance reasons. See
4661 * hmR0VmxSaveHostMsrs().
4662 *
4663 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4664 *
4665 * @returns VBox status code.
4666 * @param pVCpu Pointer to the VMCPU.
4667 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4668 * out-of-sync. Make sure to update the required fields
4669 * before using them.
4670 *
4671 * @remarks No-long-jump zone!!!
4672 */
4673static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4674{
4675 AssertPtr(pVCpu);
4676 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4677
4678 /*
4679 * MSRs that we use the auto-load/store MSR area in the VMCS.
4680 */
4681 PVM pVM = pVCpu->CTX_SUFF(pVM);
4682 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4683 {
4684 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4685#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4686 if (pVM->hm.s.fAllow64BitGuests)
4687 {
4688 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4689 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4690 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4691 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4692# ifdef DEBUG
4693 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4694 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4695 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4696# endif
4697 }
4698#endif
4699 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4700 }
4701
4702 /*
4703 * Guest Sysenter MSRs.
4704 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4705 * VM-exits on WRMSRs for these MSRs.
4706 */
4707 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4708 {
4709 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4710 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4711 }
4712
4713 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4714 {
4715 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4716 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4717 }
4718
4719 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4720 {
4721 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4722 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4723 }
4724
4725 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4726 {
4727 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4728 {
4729 /*
4730 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4731 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4732 */
4733 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4734 {
4735 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4736 AssertRCReturn(rc,rc);
4737 Log4(("Load: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pMixedCtx->msrEFER));
4738 }
4739 else
4740 {
4741 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4742 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4743 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4744 Log4(("Load: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4745 pVCpu->hm.s.vmx.cMsrs));
4746 }
4747 }
4748 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4749 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4750 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4751 }
4752
4753 return VINF_SUCCESS;
4754}
4755
4756
4757/**
4758 * Loads the guest activity state into the guest-state area in the VMCS.
4759 *
4760 * @returns VBox status code.
4761 * @param pVCpu Pointer to the VMCPU.
4762 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4763 * out-of-sync. Make sure to update the required fields
4764 * before using them.
4765 *
4766 * @remarks No-long-jump zone!!!
4767 */
4768static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4769{
4770 NOREF(pCtx);
4771 /** @todo See if we can make use of other states, e.g.
4772 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4773 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4774 {
4775 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4776 AssertRCReturn(rc, rc);
4777
4778 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4779 }
4780 return VINF_SUCCESS;
4781}
4782
4783
4784/**
4785 * Sets up the appropriate function to run guest code.
4786 *
4787 * @returns VBox status code.
4788 * @param pVCpu Pointer to the VMCPU.
4789 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4790 * out-of-sync. Make sure to update the required fields
4791 * before using them.
4792 *
4793 * @remarks No-long-jump zone!!!
4794 */
4795static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4796{
4797 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4798 {
4799#ifndef VBOX_ENABLE_64_BITS_GUESTS
4800 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4801#endif
4802 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4803#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4804 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4805 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4806 {
4807 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4808 {
4809 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4810 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4811 | HM_CHANGED_VMX_EXIT_CTLS
4812 | HM_CHANGED_VMX_ENTRY_CTLS
4813 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4814 }
4815 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4816 }
4817#else
4818 /* 64-bit host or hybrid host. */
4819 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4820#endif
4821 }
4822 else
4823 {
4824 /* Guest is not in long mode, use the 32-bit handler. */
4825#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4826 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4827 {
4828 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4829 {
4830 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4831 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4832 | HM_CHANGED_VMX_EXIT_CTLS
4833 | HM_CHANGED_VMX_ENTRY_CTLS
4834 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4835 }
4836 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4837 }
4838#else
4839 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4840#endif
4841 }
4842 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4843 return VINF_SUCCESS;
4844}
4845
4846
4847/**
4848 * Wrapper for running the guest code in VT-x.
4849 *
4850 * @returns VBox strict status code.
4851 * @param pVM Pointer to the VM.
4852 * @param pVCpu Pointer to the VMCPU.
4853 * @param pCtx Pointer to the guest-CPU context.
4854 *
4855 * @remarks No-long-jump zone!!!
4856 */
4857DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4858{
4859 /*
4860 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4861 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4862 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4863 */
4864 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4865 /** @todo Add stats for resume vs launch. */
4866#ifdef VBOX_WITH_KERNEL_USING_XMM
4867 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4868#else
4869 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4870#endif
4871}
4872
4873
4874/**
4875 * Reports world-switch error and dumps some useful debug info.
4876 *
4877 * @param pVM Pointer to the VM.
4878 * @param pVCpu Pointer to the VMCPU.
4879 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4880 * @param pCtx Pointer to the guest-CPU context.
4881 * @param pVmxTransient Pointer to the VMX transient structure (only
4882 * exitReason updated).
4883 */
4884static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4885{
4886 Assert(pVM);
4887 Assert(pVCpu);
4888 Assert(pCtx);
4889 Assert(pVmxTransient);
4890 HMVMX_ASSERT_PREEMPT_SAFE();
4891
4892 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4893 switch (rcVMRun)
4894 {
4895 case VERR_VMX_INVALID_VMXON_PTR:
4896 AssertFailed();
4897 break;
4898 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4899 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4900 {
4901 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4902 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4903 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4904 AssertRC(rc);
4905
4906 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4907 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4908 Cannot do it here as we may have been long preempted. */
4909
4910#ifdef VBOX_STRICT
4911 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4912 pVmxTransient->uExitReason));
4913 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4914 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4915 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4916 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4917 else
4918 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4919 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4920 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4921
4922 /* VMX control bits. */
4923 uint32_t u32Val;
4924 uint64_t u64Val;
4925 HMVMXHCUINTREG uHCReg;
4926 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4927 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4928 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4929 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4930 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4931 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4932 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4933 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4934 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4935 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4936 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4937 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4938 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4939 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4940 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4941 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4942 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4943 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4944 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4945 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4946 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4947 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4948 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4949 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4950 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4951 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4952 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4953 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4954 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4955 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4956 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4957 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4958 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4959 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4960 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4961 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4962 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4963 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4964 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4965 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4966 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4967 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4968
4969 /* Guest bits. */
4970 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4971 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4972 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4973 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4974 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4975 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4978
4979 /* Host bits. */
4980 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4981 Log4(("Host CR0 %#RHr\n", uHCReg));
4982 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4983 Log4(("Host CR3 %#RHr\n", uHCReg));
4984 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4985 Log4(("Host CR4 %#RHr\n", uHCReg));
4986
4987 RTGDTR HostGdtr;
4988 PCX86DESCHC pDesc;
4989 ASMGetGDTR(&HostGdtr);
4990 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4991 Log4(("Host CS %#08x\n", u32Val));
4992 if (u32Val < HostGdtr.cbGdt)
4993 {
4994 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4995 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4996 }
4997
4998 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4999 Log4(("Host DS %#08x\n", u32Val));
5000 if (u32Val < HostGdtr.cbGdt)
5001 {
5002 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5003 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5004 }
5005
5006 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5007 Log4(("Host ES %#08x\n", u32Val));
5008 if (u32Val < HostGdtr.cbGdt)
5009 {
5010 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5011 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5012 }
5013
5014 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5015 Log4(("Host FS %#08x\n", u32Val));
5016 if (u32Val < HostGdtr.cbGdt)
5017 {
5018 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5019 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5020 }
5021
5022 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5023 Log4(("Host GS %#08x\n", u32Val));
5024 if (u32Val < HostGdtr.cbGdt)
5025 {
5026 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5027 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5028 }
5029
5030 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5031 Log4(("Host SS %#08x\n", u32Val));
5032 if (u32Val < HostGdtr.cbGdt)
5033 {
5034 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5035 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5036 }
5037
5038 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5039 Log4(("Host TR %#08x\n", u32Val));
5040 if (u32Val < HostGdtr.cbGdt)
5041 {
5042 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5043 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5044 }
5045
5046 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5047 Log4(("Host TR Base %#RHv\n", uHCReg));
5048 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5049 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5050 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5051 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5052 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5053 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5054 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5055 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5056 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5057 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5058 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5059 Log4(("Host RSP %#RHv\n", uHCReg));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5061 Log4(("Host RIP %#RHv\n", uHCReg));
5062# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5063 if (HMVMX_IS_64BIT_HOST_MODE())
5064 {
5065 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5066 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5067 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5068 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5069 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5070 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5071 }
5072# endif
5073#endif /* VBOX_STRICT */
5074 break;
5075 }
5076
5077 default:
5078 /* Impossible */
5079 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5080 break;
5081 }
5082 NOREF(pVM); NOREF(pCtx);
5083}
5084
5085
5086#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5087#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5088# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5089#endif
5090#ifdef VBOX_STRICT
5091static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5092{
5093 switch (idxField)
5094 {
5095 case VMX_VMCS_GUEST_RIP:
5096 case VMX_VMCS_GUEST_RSP:
5097 case VMX_VMCS_GUEST_SYSENTER_EIP:
5098 case VMX_VMCS_GUEST_SYSENTER_ESP:
5099 case VMX_VMCS_GUEST_GDTR_BASE:
5100 case VMX_VMCS_GUEST_IDTR_BASE:
5101 case VMX_VMCS_GUEST_CS_BASE:
5102 case VMX_VMCS_GUEST_DS_BASE:
5103 case VMX_VMCS_GUEST_ES_BASE:
5104 case VMX_VMCS_GUEST_FS_BASE:
5105 case VMX_VMCS_GUEST_GS_BASE:
5106 case VMX_VMCS_GUEST_SS_BASE:
5107 case VMX_VMCS_GUEST_LDTR_BASE:
5108 case VMX_VMCS_GUEST_TR_BASE:
5109 case VMX_VMCS_GUEST_CR3:
5110 return true;
5111 }
5112 return false;
5113}
5114
5115static bool hmR0VmxIsValidReadField(uint32_t idxField)
5116{
5117 switch (idxField)
5118 {
5119 /* Read-only fields. */
5120 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5121 return true;
5122 }
5123 /* Remaining readable fields should also be writable. */
5124 return hmR0VmxIsValidWriteField(idxField);
5125}
5126#endif /* VBOX_STRICT */
5127
5128
5129/**
5130 * Executes the specified handler in 64-bit mode.
5131 *
5132 * @returns VBox status code.
5133 * @param pVM Pointer to the VM.
5134 * @param pVCpu Pointer to the VMCPU.
5135 * @param pCtx Pointer to the guest CPU context.
5136 * @param enmOp The operation to perform.
5137 * @param cbParam Number of parameters.
5138 * @param paParam Array of 32-bit parameters.
5139 */
5140VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5141 uint32_t *paParam)
5142{
5143 int rc, rc2;
5144 PHMGLOBALCPUINFO pCpu;
5145 RTHCPHYS HCPhysCpuPage;
5146 RTCCUINTREG uOldEflags;
5147
5148 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5149 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5150 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5151 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5152
5153#ifdef VBOX_STRICT
5154 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5155 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5156
5157 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5158 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5159#endif
5160
5161 /* Disable interrupts. */
5162 uOldEflags = ASMIntDisableFlags();
5163
5164#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5165 RTCPUID idHostCpu = RTMpCpuId();
5166 CPUMR0SetLApic(pVCpu, idHostCpu);
5167#endif
5168
5169 pCpu = HMR0GetCurrentCpu();
5170 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5171
5172 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5173 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5174
5175 /* Leave VMX Root Mode. */
5176 VMXDisable();
5177
5178 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5179
5180 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5181 CPUMSetHyperEIP(pVCpu, enmOp);
5182 for (int i = (int)cbParam - 1; i >= 0; i--)
5183 CPUMPushHyper(pVCpu, paParam[i]);
5184
5185 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5186
5187 /* Call the switcher. */
5188 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5189 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5190
5191 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5192 /* Make sure the VMX instructions don't cause #UD faults. */
5193 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5194
5195 /* Re-enter VMX Root Mode */
5196 rc2 = VMXEnable(HCPhysCpuPage);
5197 if (RT_FAILURE(rc2))
5198 {
5199 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5200 ASMSetFlags(uOldEflags);
5201 return rc2;
5202 }
5203
5204 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5205 AssertRC(rc2);
5206 Assert(!(ASMGetFlags() & X86_EFL_IF));
5207 ASMSetFlags(uOldEflags);
5208 return rc;
5209}
5210
5211
5212/**
5213 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5214 * supporting 64-bit guests.
5215 *
5216 * @returns VBox status code.
5217 * @param fResume Whether to VMLAUNCH or VMRESUME.
5218 * @param pCtx Pointer to the guest-CPU context.
5219 * @param pCache Pointer to the VMCS cache.
5220 * @param pVM Pointer to the VM.
5221 * @param pVCpu Pointer to the VMCPU.
5222 */
5223DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5224{
5225 uint32_t aParam[6];
5226 PHMGLOBALCPUINFO pCpu = NULL;
5227 RTHCPHYS HCPhysCpuPage = 0;
5228 int rc = VERR_INTERNAL_ERROR_5;
5229
5230 pCpu = HMR0GetCurrentCpu();
5231 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5232
5233#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5234 pCache->uPos = 1;
5235 pCache->interPD = PGMGetInterPaeCR3(pVM);
5236 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5237#endif
5238
5239#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5240 pCache->TestIn.HCPhysCpuPage = 0;
5241 pCache->TestIn.HCPhysVmcs = 0;
5242 pCache->TestIn.pCache = 0;
5243 pCache->TestOut.HCPhysVmcs = 0;
5244 pCache->TestOut.pCache = 0;
5245 pCache->TestOut.pCtx = 0;
5246 pCache->TestOut.eflags = 0;
5247#endif
5248
5249 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5250 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5251 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5252 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5253 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5254 aParam[5] = 0;
5255
5256#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5257 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5258 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5259#endif
5260 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5261
5262#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5263 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5264 Assert(pCtx->dr[4] == 10);
5265 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5266#endif
5267
5268#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5269 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5270 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5271 pVCpu->hm.s.vmx.HCPhysVmcs));
5272 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5273 pCache->TestOut.HCPhysVmcs));
5274 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5275 pCache->TestOut.pCache));
5276 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5277 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5278 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5279 pCache->TestOut.pCtx));
5280 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5281#endif
5282 return rc;
5283}
5284
5285
5286/**
5287 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5288 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5289 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5290 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5291 *
5292 * @returns VBox status code.
5293 * @param pVM Pointer to the VM.
5294 * @param pVCpu Pointer to the VMCPU.
5295 */
5296static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5297{
5298#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5299{ \
5300 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5301 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5302 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5303 ++cReadFields; \
5304}
5305
5306 AssertPtr(pVM);
5307 AssertPtr(pVCpu);
5308 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5309 uint32_t cReadFields = 0;
5310
5311 /*
5312 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5313 * and serve to indicate exceptions to the rules.
5314 */
5315
5316 /* Guest-natural selector base fields. */
5317#if 0
5318 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5319 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5320 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5321#endif
5322 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5323 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5324 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5326 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5327 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5328 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5329 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5330 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5331 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5332 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5333 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5334#if 0
5335 /* Unused natural width guest-state fields. */
5336 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5337 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5338#endif
5339 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5340 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5341
5342 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5343#if 0
5344 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5345 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5346 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5347 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5348 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5353#endif
5354
5355 /* Natural width guest-state fields. */
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5357#if 0
5358 /* Currently unused field. */
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5360#endif
5361
5362 if (pVM->hm.s.fNestedPaging)
5363 {
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5365 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5366 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5367 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5368 }
5369 else
5370 {
5371 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5372 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5373 }
5374
5375#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5376 return VINF_SUCCESS;
5377}
5378
5379
5380/**
5381 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5382 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5383 * darwin, running 64-bit guests).
5384 *
5385 * @returns VBox status code.
5386 * @param pVCpu Pointer to the VMCPU.
5387 * @param idxField The VMCS field encoding.
5388 * @param u64Val 16, 32 or 64-bit value.
5389 */
5390VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5391{
5392 int rc;
5393 switch (idxField)
5394 {
5395 /*
5396 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5397 */
5398 /* 64-bit Control fields. */
5399 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5400 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5401 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5402 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5403 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5404 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5405 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5406 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5407 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5408 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5409 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5410 case VMX_VMCS64_CTRL_EPTP_FULL:
5411 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5412 /* 64-bit Guest-state fields. */
5413 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5414 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5415 case VMX_VMCS64_GUEST_PAT_FULL:
5416 case VMX_VMCS64_GUEST_EFER_FULL:
5417 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5418 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5419 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5420 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5421 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5422 /* 64-bit Host-state fields. */
5423 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5424 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5425 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5426 {
5427 rc = VMXWriteVmcs32(idxField, u64Val);
5428 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5429 break;
5430 }
5431
5432 /*
5433 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5434 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5435 */
5436 /* Natural-width Guest-state fields. */
5437 case VMX_VMCS_GUEST_CR3:
5438 case VMX_VMCS_GUEST_ES_BASE:
5439 case VMX_VMCS_GUEST_CS_BASE:
5440 case VMX_VMCS_GUEST_SS_BASE:
5441 case VMX_VMCS_GUEST_DS_BASE:
5442 case VMX_VMCS_GUEST_FS_BASE:
5443 case VMX_VMCS_GUEST_GS_BASE:
5444 case VMX_VMCS_GUEST_LDTR_BASE:
5445 case VMX_VMCS_GUEST_TR_BASE:
5446 case VMX_VMCS_GUEST_GDTR_BASE:
5447 case VMX_VMCS_GUEST_IDTR_BASE:
5448 case VMX_VMCS_GUEST_RSP:
5449 case VMX_VMCS_GUEST_RIP:
5450 case VMX_VMCS_GUEST_SYSENTER_ESP:
5451 case VMX_VMCS_GUEST_SYSENTER_EIP:
5452 {
5453 if (!(u64Val >> 32))
5454 {
5455 /* If this field is 64-bit, VT-x will zero out the top bits. */
5456 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5457 }
5458 else
5459 {
5460 /* Assert that only the 32->64 switcher case should ever come here. */
5461 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5462 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5463 }
5464 break;
5465 }
5466
5467 default:
5468 {
5469 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5470 rc = VERR_INVALID_PARAMETER;
5471 break;
5472 }
5473 }
5474 AssertRCReturn(rc, rc);
5475 return rc;
5476}
5477
5478
5479/**
5480 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5481 * hosts (except darwin) for 64-bit guests.
5482 *
5483 * @param pVCpu Pointer to the VMCPU.
5484 * @param idxField The VMCS field encoding.
5485 * @param u64Val 16, 32 or 64-bit value.
5486 */
5487VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5488{
5489 AssertPtr(pVCpu);
5490 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5491
5492 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5493 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5494
5495 /* Make sure there are no duplicates. */
5496 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5497 {
5498 if (pCache->Write.aField[i] == idxField)
5499 {
5500 pCache->Write.aFieldVal[i] = u64Val;
5501 return VINF_SUCCESS;
5502 }
5503 }
5504
5505 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5506 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5507 pCache->Write.cValidEntries++;
5508 return VINF_SUCCESS;
5509}
5510
5511/* Enable later when the assembly code uses these as callbacks. */
5512#if 0
5513/*
5514 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5515 *
5516 * @param pVCpu Pointer to the VMCPU.
5517 * @param pCache Pointer to the VMCS cache.
5518 *
5519 * @remarks No-long-jump zone!!!
5520 */
5521VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5522{
5523 AssertPtr(pCache);
5524 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5525 {
5526 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5527 AssertRC(rc);
5528 }
5529 pCache->Write.cValidEntries = 0;
5530}
5531
5532
5533/**
5534 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5535 *
5536 * @param pVCpu Pointer to the VMCPU.
5537 * @param pCache Pointer to the VMCS cache.
5538 *
5539 * @remarks No-long-jump zone!!!
5540 */
5541VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5542{
5543 AssertPtr(pCache);
5544 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5545 {
5546 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5547 AssertRC(rc);
5548 }
5549}
5550#endif
5551#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5552
5553
5554/**
5555 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5556 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5557 * timer.
5558 *
5559 * @returns VBox status code.
5560 * @param pVCpu Pointer to the VMCPU.
5561 *
5562 * @remarks No-long-jump zone!!!
5563 */
5564static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5565{
5566 int rc = VERR_INTERNAL_ERROR_5;
5567 bool fOffsettedTsc = false;
5568 bool fParavirtTsc = false;
5569 PVM pVM = pVCpu->CTX_SUFF(pVM);
5570 if (pVM->hm.s.vmx.fUsePreemptTimer)
5571 {
5572 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5573 &pVCpu->hm.s.vmx.u64TSCOffset);
5574
5575 /* Make sure the returned values have sane upper and lower boundaries. */
5576 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5577 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5578 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5579 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5580
5581 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5582 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5583 }
5584 else
5585 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5586
5587#if 0
5588 if (fParavirtTsc)
5589 {
5590 uint64_t const u64CurTsc = ASMReadTSC();
5591 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5592 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5593 {
5594 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5595 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5596 }
5597
5598 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5599 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5600 AssertRC(rc);
5601 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5602 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5603
5604 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5605 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5606 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5607 }
5608 else
5609#endif
5610
5611 if (fParavirtTsc)
5612 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5613 if (fOffsettedTsc)
5614 {
5615 uint64_t u64CurTSC = ASMReadTSC();
5616 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5617 {
5618 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5619 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5620
5621 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5622 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5623 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5624 }
5625 else
5626 {
5627 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5628 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5629 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5630 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5631 }
5632 }
5633 else
5634 {
5635 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5636 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5637 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5638 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5639 }
5640}
5641
5642
5643/**
5644 * Determines if an exception is a contributory exception. Contributory
5645 * exceptions are ones which can cause double-faults. Page-fault is
5646 * intentionally not included here as it's a conditional contributory exception.
5647 *
5648 * @returns true if the exception is contributory, false otherwise.
5649 * @param uVector The exception vector.
5650 */
5651DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5652{
5653 switch (uVector)
5654 {
5655 case X86_XCPT_GP:
5656 case X86_XCPT_SS:
5657 case X86_XCPT_NP:
5658 case X86_XCPT_TS:
5659 case X86_XCPT_DE:
5660 return true;
5661 default:
5662 break;
5663 }
5664 return false;
5665}
5666
5667
5668/**
5669 * Sets an event as a pending event to be injected into the guest.
5670 *
5671 * @param pVCpu Pointer to the VMCPU.
5672 * @param u32IntInfo The VM-entry interruption-information field.
5673 * @param cbInstr The VM-entry instruction length in bytes (for software
5674 * interrupts, exceptions and privileged software
5675 * exceptions).
5676 * @param u32ErrCode The VM-entry exception error code.
5677 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5678 * page-fault.
5679 *
5680 * @remarks Statistics counter assumes this is a guest event being injected or
5681 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5682 * always incremented.
5683 */
5684DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5685 RTGCUINTPTR GCPtrFaultAddress)
5686{
5687 Assert(!pVCpu->hm.s.Event.fPending);
5688 pVCpu->hm.s.Event.fPending = true;
5689 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5690 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5691 pVCpu->hm.s.Event.cbInstr = cbInstr;
5692 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5693
5694 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5695}
5696
5697
5698/**
5699 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5700 *
5701 * @param pVCpu Pointer to the VMCPU.
5702 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5703 * out-of-sync. Make sure to update the required fields
5704 * before using them.
5705 */
5706DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5707{
5708 NOREF(pMixedCtx);
5709 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5710 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5711 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5712 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5713}
5714
5715
5716/**
5717 * Handle a condition that occurred while delivering an event through the guest
5718 * IDT.
5719 *
5720 * @returns VBox status code (informational error codes included).
5721 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5722 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5723 * continue execution of the guest which will delivery the #DF.
5724 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5725 *
5726 * @param pVCpu Pointer to the VMCPU.
5727 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5728 * out-of-sync. Make sure to update the required fields
5729 * before using them.
5730 * @param pVmxTransient Pointer to the VMX transient structure.
5731 *
5732 * @remarks No-long-jump zone!!!
5733 */
5734static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5735{
5736 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5737 AssertRCReturn(rc, rc);
5738 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5739 {
5740 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5741 AssertRCReturn(rc, rc);
5742
5743 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5744 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5745 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5746
5747 typedef enum
5748 {
5749 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5750 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5751 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5752 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5753 } VMXREFLECTXCPT;
5754
5755 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5756 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5757 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5758 {
5759 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5760 {
5761 enmReflect = VMXREFLECTXCPT_XCPT;
5762#ifdef VBOX_STRICT
5763 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5764 && uExitVector == X86_XCPT_PF)
5765 {
5766 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5767 }
5768#endif
5769 if ( uExitVector == X86_XCPT_PF
5770 && uIdtVector == X86_XCPT_PF)
5771 {
5772 pVmxTransient->fVectoringPF = true;
5773 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5774 }
5775 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5776 && hmR0VmxIsContributoryXcpt(uExitVector)
5777 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5778 || uIdtVector == X86_XCPT_PF))
5779 {
5780 enmReflect = VMXREFLECTXCPT_DF;
5781 }
5782 else if (uIdtVector == X86_XCPT_DF)
5783 enmReflect = VMXREFLECTXCPT_TF;
5784 }
5785 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5786 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5787 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5788 {
5789 /*
5790 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5791 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5792 */
5793 enmReflect = VMXREFLECTXCPT_XCPT;
5794 }
5795 }
5796 else
5797 {
5798 /*
5799 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5800 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5801 * original exception to the guest after handling the VM-exit.
5802 */
5803 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5804 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5805 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5806 {
5807 enmReflect = VMXREFLECTXCPT_XCPT;
5808 }
5809 }
5810
5811 switch (enmReflect)
5812 {
5813 case VMXREFLECTXCPT_XCPT:
5814 {
5815 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5816 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5817 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5818
5819 uint32_t u32ErrCode = 0;
5820 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5821 {
5822 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5823 AssertRCReturn(rc, rc);
5824 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5825 }
5826
5827 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5828 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5829 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5830 rc = VINF_SUCCESS;
5831 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5832 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5833
5834 break;
5835 }
5836
5837 case VMXREFLECTXCPT_DF:
5838 {
5839 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5840 rc = VINF_HM_DOUBLE_FAULT;
5841 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5842 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5843
5844 break;
5845 }
5846
5847 case VMXREFLECTXCPT_TF:
5848 {
5849 rc = VINF_EM_RESET;
5850 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5851 uExitVector));
5852 break;
5853 }
5854
5855 default:
5856 Assert(rc == VINF_SUCCESS);
5857 break;
5858 }
5859 }
5860 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5861 return rc;
5862}
5863
5864
5865/**
5866 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5867 *
5868 * @returns VBox status code.
5869 * @param pVCpu Pointer to the VMCPU.
5870 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5871 * out-of-sync. Make sure to update the required fields
5872 * before using them.
5873 *
5874 * @remarks No-long-jump zone!!!
5875 */
5876static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5877{
5878 NOREF(pMixedCtx);
5879
5880 /*
5881 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5882 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5883 */
5884 VMMRZCallRing3Disable(pVCpu);
5885 HM_DISABLE_PREEMPT_IF_NEEDED();
5886
5887 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5888 {
5889 uint32_t uVal = 0;
5890 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5891 AssertRCReturn(rc, rc);
5892
5893 uint32_t uShadow = 0;
5894 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5895 AssertRCReturn(rc, rc);
5896
5897 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5898 CPUMSetGuestCR0(pVCpu, uVal);
5899 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5900 }
5901
5902 HM_RESTORE_PREEMPT_IF_NEEDED();
5903 VMMRZCallRing3Enable(pVCpu);
5904 return VINF_SUCCESS;
5905}
5906
5907
5908/**
5909 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5910 *
5911 * @returns VBox status code.
5912 * @param pVCpu Pointer to the VMCPU.
5913 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5914 * out-of-sync. Make sure to update the required fields
5915 * before using them.
5916 *
5917 * @remarks No-long-jump zone!!!
5918 */
5919static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5920{
5921 NOREF(pMixedCtx);
5922
5923 int rc = VINF_SUCCESS;
5924 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5925 {
5926 uint32_t uVal = 0;
5927 uint32_t uShadow = 0;
5928 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5929 AssertRCReturn(rc, rc);
5930 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5931 AssertRCReturn(rc, rc);
5932
5933 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5934 CPUMSetGuestCR4(pVCpu, uVal);
5935 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5936 }
5937 return rc;
5938}
5939
5940
5941/**
5942 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5943 *
5944 * @returns VBox status code.
5945 * @param pVCpu Pointer to the VMCPU.
5946 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5947 * out-of-sync. Make sure to update the required fields
5948 * before using them.
5949 *
5950 * @remarks No-long-jump zone!!!
5951 */
5952static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5953{
5954 int rc = VINF_SUCCESS;
5955 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5956 {
5957 uint64_t u64Val = 0;
5958 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5959 AssertRCReturn(rc, rc);
5960
5961 pMixedCtx->rip = u64Val;
5962 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5963 }
5964 return rc;
5965}
5966
5967
5968/**
5969 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5970 *
5971 * @returns VBox status code.
5972 * @param pVCpu Pointer to the VMCPU.
5973 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5974 * out-of-sync. Make sure to update the required fields
5975 * before using them.
5976 *
5977 * @remarks No-long-jump zone!!!
5978 */
5979static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5980{
5981 int rc = VINF_SUCCESS;
5982 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5983 {
5984 uint64_t u64Val = 0;
5985 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5986 AssertRCReturn(rc, rc);
5987
5988 pMixedCtx->rsp = u64Val;
5989 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5990 }
5991 return rc;
5992}
5993
5994
5995/**
5996 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5997 *
5998 * @returns VBox status code.
5999 * @param pVCpu Pointer to the VMCPU.
6000 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6001 * out-of-sync. Make sure to update the required fields
6002 * before using them.
6003 *
6004 * @remarks No-long-jump zone!!!
6005 */
6006static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6007{
6008 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6009 {
6010 uint32_t uVal = 0;
6011 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6012 AssertRCReturn(rc, rc);
6013
6014 pMixedCtx->eflags.u32 = uVal;
6015 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6016 {
6017 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6018 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6019
6020 pMixedCtx->eflags.Bits.u1VM = 0;
6021 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6022 }
6023
6024 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6025 }
6026 return VINF_SUCCESS;
6027}
6028
6029
6030/**
6031 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6032 * guest-CPU context.
6033 */
6034DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6035{
6036 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6037 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6038 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6039 return rc;
6040}
6041
6042
6043/**
6044 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6045 * from the guest-state area in the VMCS.
6046 *
6047 * @param pVCpu Pointer to the VMCPU.
6048 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6049 * out-of-sync. Make sure to update the required fields
6050 * before using them.
6051 *
6052 * @remarks No-long-jump zone!!!
6053 */
6054static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6055{
6056 uint32_t uIntrState = 0;
6057 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6058 AssertRC(rc);
6059
6060 if (!uIntrState)
6061 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6062 else
6063 {
6064 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
6065 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6066 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6067 AssertRC(rc);
6068 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6069 AssertRC(rc);
6070
6071 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6072 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6073 }
6074}
6075
6076
6077/**
6078 * Saves the guest's activity state.
6079 *
6080 * @returns VBox status code.
6081 * @param pVCpu Pointer to the VMCPU.
6082 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6083 * out-of-sync. Make sure to update the required fields
6084 * before using them.
6085 *
6086 * @remarks No-long-jump zone!!!
6087 */
6088static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6089{
6090 NOREF(pMixedCtx);
6091 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6092 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6093 return VINF_SUCCESS;
6094}
6095
6096
6097/**
6098 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6099 * the current VMCS into the guest-CPU context.
6100 *
6101 * @returns VBox status code.
6102 * @param pVCpu Pointer to the VMCPU.
6103 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6104 * out-of-sync. Make sure to update the required fields
6105 * before using them.
6106 *
6107 * @remarks No-long-jump zone!!!
6108 */
6109static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6110{
6111 int rc = VINF_SUCCESS;
6112 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6113 {
6114 uint32_t u32Val = 0;
6115 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6116 pMixedCtx->SysEnter.cs = u32Val;
6117 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6118 }
6119
6120 uint64_t u64Val = 0;
6121 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6122 {
6123 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6124 pMixedCtx->SysEnter.eip = u64Val;
6125 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6126 }
6127 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6128 {
6129 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6130 pMixedCtx->SysEnter.esp = u64Val;
6131 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6132 }
6133 return rc;
6134}
6135
6136
6137/**
6138 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6139 * the CPU back into the guest-CPU context.
6140 *
6141 * @returns VBox status code.
6142 * @param pVCpu Pointer to the VMCPU.
6143 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6144 * out-of-sync. Make sure to update the required fields
6145 * before using them.
6146 *
6147 * @remarks No-long-jump zone!!!
6148 */
6149static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6150{
6151#if HC_ARCH_BITS == 64
6152 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6153 {
6154 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6155 VMMRZCallRing3Disable(pVCpu);
6156 HM_DISABLE_PREEMPT_IF_NEEDED();
6157
6158 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6159 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6160 {
6161 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6162 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6163 }
6164
6165 HM_RESTORE_PREEMPT_IF_NEEDED();
6166 VMMRZCallRing3Enable(pVCpu);
6167 }
6168 else
6169 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6170#else
6171 NOREF(pMixedCtx);
6172 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6173#endif
6174
6175 return VINF_SUCCESS;
6176}
6177
6178
6179/**
6180 * Saves the auto load/store'd guest MSRs from the current VMCS into
6181 * the guest-CPU context.
6182 *
6183 * @returns VBox status code.
6184 * @param pVCpu Pointer to the VMCPU.
6185 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6186 * out-of-sync. Make sure to update the required fields
6187 * before using them.
6188 *
6189 * @remarks No-long-jump zone!!!
6190 */
6191static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6192{
6193 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6194 return VINF_SUCCESS;
6195
6196 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6197 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6198 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6199 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6200 {
6201 switch (pMsr->u32Msr)
6202 {
6203 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6204 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6205 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6206 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6207 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6208 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6209 break;
6210
6211 default:
6212 {
6213 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6214 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6215 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6216 }
6217 }
6218 }
6219
6220 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6221 return VINF_SUCCESS;
6222}
6223
6224
6225/**
6226 * Saves the guest control registers from the current VMCS into the guest-CPU
6227 * context.
6228 *
6229 * @returns VBox status code.
6230 * @param pVCpu Pointer to the VMCPU.
6231 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6232 * out-of-sync. Make sure to update the required fields
6233 * before using them.
6234 *
6235 * @remarks No-long-jump zone!!!
6236 */
6237static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6238{
6239 /* Guest CR0. Guest FPU. */
6240 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6241 AssertRCReturn(rc, rc);
6242
6243 /* Guest CR4. */
6244 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6245 AssertRCReturn(rc, rc);
6246
6247 /* Guest CR2 - updated always during the world-switch or in #PF. */
6248 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6249 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6250 {
6251 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6252 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6253
6254 PVM pVM = pVCpu->CTX_SUFF(pVM);
6255 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6256 || ( pVM->hm.s.fNestedPaging
6257 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6258 {
6259 uint64_t u64Val = 0;
6260 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6261 if (pMixedCtx->cr3 != u64Val)
6262 {
6263 CPUMSetGuestCR3(pVCpu, u64Val);
6264 if (VMMRZCallRing3IsEnabled(pVCpu))
6265 {
6266 PGMUpdateCR3(pVCpu, u64Val);
6267 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6268 }
6269 else
6270 {
6271 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6272 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6273 }
6274 }
6275
6276 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6277 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6278 {
6279 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6280 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6281 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6282 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6283
6284 if (VMMRZCallRing3IsEnabled(pVCpu))
6285 {
6286 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6287 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6288 }
6289 else
6290 {
6291 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6292 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6293 }
6294 }
6295 }
6296
6297 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6298 }
6299
6300 /*
6301 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6302 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6303 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6304 *
6305 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6306 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6307 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6308 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6309 *
6310 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6311 */
6312 if (VMMRZCallRing3IsEnabled(pVCpu))
6313 {
6314 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6315 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6316
6317 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6318 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6319
6320 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6321 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6322 }
6323
6324 return rc;
6325}
6326
6327
6328/**
6329 * Reads a guest segment register from the current VMCS into the guest-CPU
6330 * context.
6331 *
6332 * @returns VBox status code.
6333 * @param pVCpu Pointer to the VMCPU.
6334 * @param idxSel Index of the selector in the VMCS.
6335 * @param idxLimit Index of the segment limit in the VMCS.
6336 * @param idxBase Index of the segment base in the VMCS.
6337 * @param idxAccess Index of the access rights of the segment in the VMCS.
6338 * @param pSelReg Pointer to the segment selector.
6339 *
6340 * @remarks No-long-jump zone!!!
6341 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6342 * macro as that takes care of whether to read from the VMCS cache or
6343 * not.
6344 */
6345DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6346 PCPUMSELREG pSelReg)
6347{
6348 NOREF(pVCpu);
6349
6350 uint32_t u32Val = 0;
6351 int rc = VMXReadVmcs32(idxSel, &u32Val);
6352 AssertRCReturn(rc, rc);
6353 pSelReg->Sel = (uint16_t)u32Val;
6354 pSelReg->ValidSel = (uint16_t)u32Val;
6355 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6356
6357 rc = VMXReadVmcs32(idxLimit, &u32Val);
6358 AssertRCReturn(rc, rc);
6359 pSelReg->u32Limit = u32Val;
6360
6361 uint64_t u64Val = 0;
6362 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6363 AssertRCReturn(rc, rc);
6364 pSelReg->u64Base = u64Val;
6365
6366 rc = VMXReadVmcs32(idxAccess, &u32Val);
6367 AssertRCReturn(rc, rc);
6368 pSelReg->Attr.u = u32Val;
6369
6370 /*
6371 * If VT-x marks the segment as unusable, most other bits remain undefined:
6372 * - For CS the L, D and G bits have meaning.
6373 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6374 * - For the remaining data segments no bits are defined.
6375 *
6376 * The present bit and the unusable bit has been observed to be set at the
6377 * same time (the selector was supposed to invalid as we started executing
6378 * a V8086 interrupt in ring-0).
6379 *
6380 * What should be important for the rest of the VBox code, is that the P bit is
6381 * cleared. Some of the other VBox code recognizes the unusable bit, but
6382 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6383 * safe side here, we'll strip off P and other bits we don't care about. If
6384 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6385 *
6386 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6387 */
6388 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6389 {
6390 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6391
6392 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6393 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6394 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6395
6396 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6397#ifdef DEBUG_bird
6398 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6399 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6400 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6401#endif
6402 }
6403 return VINF_SUCCESS;
6404}
6405
6406
6407#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6408# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6409 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6410 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6411#else
6412# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6413 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6414 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6415#endif
6416
6417
6418/**
6419 * Saves the guest segment registers from the current VMCS into the guest-CPU
6420 * context.
6421 *
6422 * @returns VBox status code.
6423 * @param pVCpu Pointer to the VMCPU.
6424 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6425 * out-of-sync. Make sure to update the required fields
6426 * before using them.
6427 *
6428 * @remarks No-long-jump zone!!!
6429 */
6430static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6431{
6432 /* Guest segment registers. */
6433 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6434 {
6435 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6436 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6437 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6438 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6439 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6440 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6441 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6442
6443 /* Restore segment attributes for real-on-v86 mode hack. */
6444 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6445 {
6446 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6447 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6448 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6449 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6450 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6451 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6452 }
6453 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6454 }
6455
6456 return VINF_SUCCESS;
6457}
6458
6459
6460/**
6461 * Saves the guest descriptor table registers and task register from the current
6462 * VMCS into the guest-CPU context.
6463 *
6464 * @returns VBox status code.
6465 * @param pVCpu Pointer to the VMCPU.
6466 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6467 * out-of-sync. Make sure to update the required fields
6468 * before using them.
6469 *
6470 * @remarks No-long-jump zone!!!
6471 */
6472static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6473{
6474 int rc = VINF_SUCCESS;
6475
6476 /* Guest LDTR. */
6477 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6478 {
6479 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6480 AssertRCReturn(rc, rc);
6481 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6482 }
6483
6484 /* Guest GDTR. */
6485 uint64_t u64Val = 0;
6486 uint32_t u32Val = 0;
6487 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6488 {
6489 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6490 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6491 pMixedCtx->gdtr.pGdt = u64Val;
6492 pMixedCtx->gdtr.cbGdt = u32Val;
6493 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6494 }
6495
6496 /* Guest IDTR. */
6497 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6498 {
6499 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6500 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6501 pMixedCtx->idtr.pIdt = u64Val;
6502 pMixedCtx->idtr.cbIdt = u32Val;
6503 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6504 }
6505
6506 /* Guest TR. */
6507 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6508 {
6509 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6510 AssertRCReturn(rc, rc);
6511
6512 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6513 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6514 {
6515 rc = VMXLOCAL_READ_SEG(TR, tr);
6516 AssertRCReturn(rc, rc);
6517 }
6518 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6519 }
6520 return rc;
6521}
6522
6523#undef VMXLOCAL_READ_SEG
6524
6525
6526/**
6527 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6528 * context.
6529 *
6530 * @returns VBox status code.
6531 * @param pVCpu Pointer to the VMCPU.
6532 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6533 * out-of-sync. Make sure to update the required fields
6534 * before using them.
6535 *
6536 * @remarks No-long-jump zone!!!
6537 */
6538static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6539{
6540 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6541 {
6542 if (!pVCpu->hm.s.fUsingHyperDR7)
6543 {
6544 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6545 uint32_t u32Val;
6546 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6547 pMixedCtx->dr[7] = u32Val;
6548 }
6549
6550 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6551 }
6552 return VINF_SUCCESS;
6553}
6554
6555
6556/**
6557 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6558 *
6559 * @returns VBox status code.
6560 * @param pVCpu Pointer to the VMCPU.
6561 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6562 * out-of-sync. Make sure to update the required fields
6563 * before using them.
6564 *
6565 * @remarks No-long-jump zone!!!
6566 */
6567static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6568{
6569 NOREF(pMixedCtx);
6570
6571 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6572 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6573 return VINF_SUCCESS;
6574}
6575
6576
6577/**
6578 * Saves the entire guest state from the currently active VMCS into the
6579 * guest-CPU context. This essentially VMREADs all guest-data.
6580 *
6581 * @returns VBox status code.
6582 * @param pVCpu Pointer to the VMCPU.
6583 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6584 * out-of-sync. Make sure to update the required fields
6585 * before using them.
6586 */
6587static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6588{
6589 Assert(pVCpu);
6590 Assert(pMixedCtx);
6591
6592 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6593 return VINF_SUCCESS;
6594
6595 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6596 again on the ring-3 callback path, there is no real need to. */
6597 if (VMMRZCallRing3IsEnabled(pVCpu))
6598 VMMR0LogFlushDisable(pVCpu);
6599 else
6600 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6601 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6602
6603 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6604 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6605
6606 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6607 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6608
6609 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6610 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6611
6612 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6613 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6614
6615 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6616 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6617
6618 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6619 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6620
6621 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6622 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6623
6624 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6625 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6626
6627 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6628 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6629
6630 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6631 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6632
6633 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6634 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6635
6636 if (VMMRZCallRing3IsEnabled(pVCpu))
6637 VMMR0LogFlushEnable(pVCpu);
6638
6639 return rc;
6640}
6641
6642
6643/**
6644 * Check per-VM and per-VCPU force flag actions that require us to go back to
6645 * ring-3 for one reason or another.
6646 *
6647 * @returns VBox status code (information status code included).
6648 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6649 * ring-3.
6650 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6651 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6652 * interrupts)
6653 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6654 * all EMTs to be in ring-3.
6655 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6656 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6657 * to the EM loop.
6658 *
6659 * @param pVM Pointer to the VM.
6660 * @param pVCpu Pointer to the VMCPU.
6661 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6662 * out-of-sync. Make sure to update the required fields
6663 * before using them.
6664 */
6665static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6666{
6667 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6668
6669 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6670 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6671 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6672 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6673 {
6674 /* We need the control registers now, make sure the guest-CPU context is updated. */
6675 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6676 AssertRCReturn(rc3, rc3);
6677
6678 /* Pending HM CR3 sync. */
6679 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6680 {
6681 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6682 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6683 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6684 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6685 }
6686
6687 /* Pending HM PAE PDPEs. */
6688 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6689 {
6690 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6691 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6692 }
6693
6694 /* Pending PGM C3 sync. */
6695 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6696 {
6697 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6698 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6699 if (rc2 != VINF_SUCCESS)
6700 {
6701 AssertRC(rc2);
6702 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6703 return rc2;
6704 }
6705 }
6706
6707 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6708 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6709 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6710 {
6711 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6712 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6713 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6714 return rc2;
6715 }
6716
6717 /* Pending VM request packets, such as hardware interrupts. */
6718 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6719 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6720 {
6721 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6722 return VINF_EM_PENDING_REQUEST;
6723 }
6724
6725 /* Pending PGM pool flushes. */
6726 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6727 {
6728 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6729 return VINF_PGM_POOL_FLUSH_PENDING;
6730 }
6731
6732 /* Pending DMA requests. */
6733 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6734 {
6735 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6736 return VINF_EM_RAW_TO_R3;
6737 }
6738 }
6739
6740 return VINF_SUCCESS;
6741}
6742
6743
6744/**
6745 * Converts any TRPM trap into a pending HM event. This is typically used when
6746 * entering from ring-3 (not longjmp returns).
6747 *
6748 * @param pVCpu Pointer to the VMCPU.
6749 */
6750static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6751{
6752 Assert(TRPMHasTrap(pVCpu));
6753 Assert(!pVCpu->hm.s.Event.fPending);
6754
6755 uint8_t uVector;
6756 TRPMEVENT enmTrpmEvent;
6757 RTGCUINT uErrCode;
6758 RTGCUINTPTR GCPtrFaultAddress;
6759 uint8_t cbInstr;
6760
6761 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6762 AssertRC(rc);
6763
6764 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6765 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6766 if (enmTrpmEvent == TRPM_TRAP)
6767 {
6768 switch (uVector)
6769 {
6770 case X86_XCPT_BP:
6771 case X86_XCPT_OF:
6772 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6773 break;
6774
6775 case X86_XCPT_PF:
6776 case X86_XCPT_DF:
6777 case X86_XCPT_TS:
6778 case X86_XCPT_NP:
6779 case X86_XCPT_SS:
6780 case X86_XCPT_GP:
6781 case X86_XCPT_AC:
6782 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6783 /* no break! */
6784 default:
6785 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6786 break;
6787 }
6788 }
6789 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6790 {
6791 if (uVector == X86_XCPT_NMI)
6792 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6793 else
6794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6795 }
6796 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6797 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6798 else
6799 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6800
6801 rc = TRPMResetTrap(pVCpu);
6802 AssertRC(rc);
6803 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6804 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6805
6806 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6807 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6808}
6809
6810
6811/**
6812 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6813 * VT-x to execute any instruction.
6814 *
6815 * @param pvCpu Pointer to the VMCPU.
6816 */
6817static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6818{
6819 Assert(pVCpu->hm.s.Event.fPending);
6820
6821 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6822 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6823 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6824 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6825
6826 /* If a trap was already pending, we did something wrong! */
6827 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6828
6829 TRPMEVENT enmTrapType;
6830 switch (uVectorType)
6831 {
6832 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6833 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6834 enmTrapType = TRPM_HARDWARE_INT;
6835 break;
6836
6837 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6838 enmTrapType = TRPM_SOFTWARE_INT;
6839 break;
6840
6841 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6842 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6843 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6844 enmTrapType = TRPM_TRAP;
6845 break;
6846
6847 default:
6848 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6849 enmTrapType = TRPM_32BIT_HACK;
6850 break;
6851 }
6852
6853 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6854
6855 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6856 AssertRC(rc);
6857
6858 if (fErrorCodeValid)
6859 TRPMSetErrorCode(pVCpu, uErrorCode);
6860
6861 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6862 && uVector == X86_XCPT_PF)
6863 {
6864 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6865 }
6866 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6867 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6868 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6869 {
6870 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6871 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6872 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6873 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6874 }
6875 pVCpu->hm.s.Event.fPending = false;
6876}
6877
6878
6879/**
6880 * Does the necessary state syncing before returning to ring-3 for any reason
6881 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6882 *
6883 * @returns VBox status code.
6884 * @param pVM Pointer to the VM.
6885 * @param pVCpu Pointer to the VMCPU.
6886 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6887 * be out-of-sync. Make sure to update the required
6888 * fields before using them.
6889 * @param fSaveGuestState Whether to save the guest state or not.
6890 *
6891 * @remarks No-long-jmp zone!!!
6892 */
6893static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6894{
6895 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6896 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6897
6898 RTCPUID idCpu = RTMpCpuId();
6899 Log4Func(("HostCpuId=%u\n", idCpu));
6900
6901 /*
6902 * !!! IMPORTANT !!!
6903 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6904 */
6905
6906 /* Save the guest state if necessary. */
6907 if ( fSaveGuestState
6908 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6909 {
6910 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6911 AssertRCReturn(rc, rc);
6912 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6913 }
6914
6915 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6916 if (CPUMIsGuestFPUStateActive(pVCpu))
6917 {
6918 /* We shouldn't reload CR0 without saving it first. */
6919 if (!fSaveGuestState)
6920 {
6921 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6922 AssertRCReturn(rc, rc);
6923 }
6924 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6925 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6926 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6927 }
6928
6929 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6930#ifdef VBOX_STRICT
6931 if (CPUMIsHyperDebugStateActive(pVCpu))
6932 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6933#endif
6934 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6935 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6936 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6937 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6938
6939#if HC_ARCH_BITS == 64
6940 /* Restore host-state bits that VT-x only restores partially. */
6941 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6942 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6943 {
6944 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6945 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6946 }
6947 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6948#endif
6949
6950#if HC_ARCH_BITS == 64
6951 /* Restore the host MSRs as we're leaving VT-x context. */
6952 if ( pVM->hm.s.fAllow64BitGuests
6953 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6954 {
6955 /* We shouldn't reload the guest MSRs without saving it first. */
6956 if (!fSaveGuestState)
6957 {
6958 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6959 AssertRCReturn(rc, rc);
6960 }
6961 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6962 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6963 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6964 }
6965#endif
6966
6967 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6968 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6969
6970 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6971 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6972 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6973 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6974 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6975 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6976 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6977 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6978
6979 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6980
6981 /** @todo This partially defeats the purpose of having preemption hooks.
6982 * The problem is, deregistering the hooks should be moved to a place that
6983 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6984 * context.
6985 */
6986 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6987 {
6988 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6989 AssertRCReturn(rc, rc);
6990
6991 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6992 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6993 }
6994 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6995 NOREF(idCpu);
6996
6997 return VINF_SUCCESS;
6998}
6999
7000
7001/**
7002 * Leaves the VT-x session.
7003 *
7004 * @returns VBox status code.
7005 * @param pVM Pointer to the VM.
7006 * @param pVCpu Pointer to the VMCPU.
7007 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7008 * out-of-sync. Make sure to update the required fields
7009 * before using them.
7010 *
7011 * @remarks No-long-jmp zone!!!
7012 */
7013DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7014{
7015 HM_DISABLE_PREEMPT_IF_NEEDED();
7016 HMVMX_ASSERT_CPU_SAFE();
7017 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7018 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7019
7020 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7021 and done this from the VMXR0ThreadCtxCallback(). */
7022 if (!pVCpu->hm.s.fLeaveDone)
7023 {
7024 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7025 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7026 pVCpu->hm.s.fLeaveDone = true;
7027 }
7028
7029 /*
7030 * !!! IMPORTANT !!!
7031 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7032 */
7033
7034 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7035 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7036 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7037 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7038 VMMR0ThreadCtxHooksDeregister(pVCpu);
7039
7040 /* Leave HM context. This takes care of local init (term). */
7041 int rc = HMR0LeaveCpu(pVCpu);
7042
7043 HM_RESTORE_PREEMPT_IF_NEEDED();
7044
7045 return rc;
7046}
7047
7048
7049/**
7050 * Does the necessary state syncing before doing a longjmp to ring-3.
7051 *
7052 * @returns VBox status code.
7053 * @param pVM Pointer to the VM.
7054 * @param pVCpu Pointer to the VMCPU.
7055 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7056 * out-of-sync. Make sure to update the required fields
7057 * before using them.
7058 *
7059 * @remarks No-long-jmp zone!!!
7060 */
7061DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7062{
7063 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7064}
7065
7066
7067/**
7068 * Take necessary actions before going back to ring-3.
7069 *
7070 * An action requires us to go back to ring-3. This function does the necessary
7071 * steps before we can safely return to ring-3. This is not the same as longjmps
7072 * to ring-3, this is voluntary and prepares the guest so it may continue
7073 * executing outside HM (recompiler/IEM).
7074 *
7075 * @returns VBox status code.
7076 * @param pVM Pointer to the VM.
7077 * @param pVCpu Pointer to the VMCPU.
7078 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7079 * out-of-sync. Make sure to update the required fields
7080 * before using them.
7081 * @param rcExit The reason for exiting to ring-3. Can be
7082 * VINF_VMM_UNKNOWN_RING3_CALL.
7083 */
7084static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7085{
7086 Assert(pVM);
7087 Assert(pVCpu);
7088 Assert(pMixedCtx);
7089 HMVMX_ASSERT_PREEMPT_SAFE();
7090
7091 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7092 {
7093 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7094 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7095 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7096 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7097 }
7098
7099 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7100 VMMRZCallRing3Disable(pVCpu);
7101 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7102
7103 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7104 if (pVCpu->hm.s.Event.fPending)
7105 {
7106 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7107 Assert(!pVCpu->hm.s.Event.fPending);
7108 }
7109
7110 /* Save guest state and restore host state bits. */
7111 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7112 AssertRCReturn(rc, rc);
7113 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7114
7115 /* Sync recompiler state. */
7116 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7117 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7118 | CPUM_CHANGED_LDTR
7119 | CPUM_CHANGED_GDTR
7120 | CPUM_CHANGED_IDTR
7121 | CPUM_CHANGED_TR
7122 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7123 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7124 if ( pVM->hm.s.fNestedPaging
7125 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7126 {
7127 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7128 }
7129
7130 Assert(!pVCpu->hm.s.fClearTrapFlag);
7131
7132 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7133 if (rcExit != VINF_EM_RAW_INTERRUPT)
7134 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7135
7136 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7137
7138 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7139 VMMRZCallRing3RemoveNotification(pVCpu);
7140 VMMRZCallRing3Enable(pVCpu);
7141
7142 return rc;
7143}
7144
7145
7146/**
7147 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7148 * longjump to ring-3 and possibly get preempted.
7149 *
7150 * @returns VBox status code.
7151 * @param pVCpu Pointer to the VMCPU.
7152 * @param enmOperation The operation causing the ring-3 longjump.
7153 * @param pvUser Opaque pointer to the guest-CPU context. The data
7154 * may be out-of-sync. Make sure to update the required
7155 * fields before using them.
7156 */
7157DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7158{
7159 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7160 {
7161 /*
7162 * !!! IMPORTANT !!!
7163 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7164 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7165 */
7166 VMMRZCallRing3RemoveNotification(pVCpu);
7167 VMMRZCallRing3Disable(pVCpu);
7168 HM_DISABLE_PREEMPT_IF_NEEDED();
7169
7170 PVM pVM = pVCpu->CTX_SUFF(pVM);
7171 if (CPUMIsGuestFPUStateActive(pVCpu))
7172 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7173
7174 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7175
7176#if HC_ARCH_BITS == 64
7177 /* Restore host-state bits that VT-x only restores partially. */
7178 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7179 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7180 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7181 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7182
7183 /* Restore the host MSRs as we're leaving VT-x context. */
7184 if ( pVM->hm.s.fAllow64BitGuests
7185 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7186 {
7187 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7188 }
7189#endif
7190 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7191 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7192 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7193 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7194 {
7195 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7196 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7197 }
7198
7199 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7200 VMMR0ThreadCtxHooksDeregister(pVCpu);
7201
7202 HMR0LeaveCpu(pVCpu);
7203 HM_RESTORE_PREEMPT_IF_NEEDED();
7204 return VINF_SUCCESS;
7205 }
7206
7207 Assert(pVCpu);
7208 Assert(pvUser);
7209 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7210 HMVMX_ASSERT_PREEMPT_SAFE();
7211
7212 VMMRZCallRing3Disable(pVCpu);
7213 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7214
7215 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7216 enmOperation));
7217
7218 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7219 AssertRCReturn(rc, rc);
7220
7221 VMMRZCallRing3Enable(pVCpu);
7222 return VINF_SUCCESS;
7223}
7224
7225
7226/**
7227 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7228 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7229 *
7230 * @param pVCpu Pointer to the VMCPU.
7231 */
7232DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7233{
7234 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7235 {
7236 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7237 {
7238 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7239 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7240 AssertRC(rc);
7241 Log4(("Setup interrupt-window exiting\n"));
7242 }
7243 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7244}
7245
7246
7247/**
7248 * Clears the interrupt-window exiting control in the VMCS.
7249 *
7250 * @param pVCpu Pointer to the VMCPU.
7251 */
7252DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7253{
7254 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7255 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7256 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7257 AssertRC(rc);
7258 Log4(("Cleared interrupt-window exiting\n"));
7259}
7260
7261
7262/**
7263 * Evaluates the event to be delivered to the guest and sets it as the pending
7264 * event.
7265 *
7266 * @param pVCpu Pointer to the VMCPU.
7267 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7268 * out-of-sync. Make sure to update the required fields
7269 * before using them.
7270 */
7271static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7272{
7273 Assert(!pVCpu->hm.s.Event.fPending);
7274
7275 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7276 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7277 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7278 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7279
7280 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7281 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7282 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7283 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7284 Assert(!TRPMHasTrap(pVCpu));
7285
7286 /** @todo SMI. SMIs take priority over NMIs. */
7287 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7288 {
7289 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7290 if ( !fBlockMovSS
7291 && !fBlockSti)
7292 {
7293 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7294 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7295 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7296 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7297
7298 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7299 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7300 }
7301 else
7302 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7303 }
7304 /*
7305 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7306 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7307 */
7308 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7309 && !pVCpu->hm.s.fSingleInstruction)
7310 {
7311 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7312 AssertRC(rc);
7313 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7314 if ( !fBlockInt
7315 && !fBlockSti
7316 && !fBlockMovSS)
7317 {
7318 uint8_t u8Interrupt;
7319 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7320 if (RT_SUCCESS(rc))
7321 {
7322 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7323 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7324 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7325
7326 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7327 }
7328 else
7329 {
7330 /** @todo Does this actually happen? If not turn it into an assertion. */
7331 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7332 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7333 }
7334 }
7335 else
7336 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7337 }
7338}
7339
7340
7341/**
7342 * Sets a pending-debug exception to be delivered to the guest if the guest is
7343 * single-stepping.
7344 *
7345 * @param pVCpu Pointer to the VMCPU.
7346 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7347 * out-of-sync. Make sure to update the required fields
7348 * before using them.
7349 */
7350DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7351{
7352 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7353 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7354 {
7355 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7356 AssertRC(rc);
7357 }
7358}
7359
7360
7361/**
7362 * Injects any pending events into the guest if the guest is in a state to
7363 * receive them.
7364 *
7365 * @returns VBox status code (informational status codes included).
7366 * @param pVCpu Pointer to the VMCPU.
7367 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7368 * out-of-sync. Make sure to update the required fields
7369 * before using them.
7370 */
7371static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7372{
7373 HMVMX_ASSERT_PREEMPT_SAFE();
7374 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7375
7376 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7377 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7378 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7379 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7380
7381 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7382 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7383 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7384 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7385 Assert(!TRPMHasTrap(pVCpu));
7386
7387 int rc = VINF_SUCCESS;
7388 if (pVCpu->hm.s.Event.fPending)
7389 {
7390 /*
7391 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7392 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7393 * ended up enabling interrupts outside VT-x.
7394 */
7395 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7396 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7397 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7398 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7399 {
7400 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7401 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7402 }
7403#if defined(VBOX_STRICT)
7404 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7405 {
7406 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7407 AssertRCReturn(rc, rc);
7408 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7409 Assert(!fBlockInt);
7410 Assert(!fBlockSti);
7411 Assert(!fBlockMovSS);
7412 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7413 }
7414 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7415 {
7416 Assert(!fBlockSti);
7417 Assert(!fBlockMovSS);
7418 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7419 }
7420#endif
7421 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7422 (uint8_t)uIntType));
7423 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7424 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7425 AssertRCReturn(rc, rc);
7426
7427 /* Update the interruptibility-state as it could have been changed by
7428 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7429 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7430 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7431
7432#ifdef VBOX_WITH_STATISTICS
7433 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7434 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7435 else
7436 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7437#endif
7438 }
7439
7440 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7441 if ( fBlockSti
7442 || fBlockMovSS)
7443 {
7444 if ( !pVCpu->hm.s.fSingleInstruction
7445 && !DBGFIsStepping(pVCpu))
7446 {
7447 /*
7448 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7449 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7450 * See Intel spec. 27.3.4 "Saving Non-Register State".
7451 */
7452 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7453 AssertRCReturn(rc2, rc2);
7454 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7455 }
7456 else if (pMixedCtx->eflags.Bits.u1TF)
7457 {
7458 /*
7459 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7460 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7461 */
7462 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7463 uIntrState = 0;
7464 }
7465 }
7466
7467 /*
7468 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7469 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7470 */
7471 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7472 AssertRC(rc2);
7473
7474 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7475 NOREF(fBlockMovSS); NOREF(fBlockSti);
7476 return rc;
7477}
7478
7479
7480/**
7481 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7482 *
7483 * @param pVCpu Pointer to the VMCPU.
7484 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7485 * out-of-sync. Make sure to update the required fields
7486 * before using them.
7487 */
7488DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7489{
7490 NOREF(pMixedCtx);
7491 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7492 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7493}
7494
7495
7496/**
7497 * Injects a double-fault (#DF) exception into the VM.
7498 *
7499 * @returns VBox status code (informational status code included).
7500 * @param pVCpu Pointer to the VMCPU.
7501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7502 * out-of-sync. Make sure to update the required fields
7503 * before using them.
7504 */
7505DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7506{
7507 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7508 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7509 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7510 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7511 puIntrState);
7512}
7513
7514
7515/**
7516 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7517 *
7518 * @param pVCpu Pointer to the VMCPU.
7519 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7520 * out-of-sync. Make sure to update the required fields
7521 * before using them.
7522 */
7523DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7524{
7525 NOREF(pMixedCtx);
7526 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7527 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7528 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7529}
7530
7531
7532/**
7533 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7534 *
7535 * @param pVCpu Pointer to the VMCPU.
7536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7537 * out-of-sync. Make sure to update the required fields
7538 * before using them.
7539 * @param cbInstr The value of RIP that is to be pushed on the guest
7540 * stack.
7541 */
7542DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7543{
7544 NOREF(pMixedCtx);
7545 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7546 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7547 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7548}
7549
7550
7551/**
7552 * Injects a general-protection (#GP) fault into the VM.
7553 *
7554 * @returns VBox status code (informational status code included).
7555 * @param pVCpu Pointer to the VMCPU.
7556 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7557 * out-of-sync. Make sure to update the required fields
7558 * before using them.
7559 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7560 * mode, i.e. in real-mode it's not valid).
7561 * @param u32ErrorCode The error code associated with the #GP.
7562 */
7563DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7564 uint32_t *puIntrState)
7565{
7566 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7567 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7568 if (fErrorCodeValid)
7569 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7570 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7571 puIntrState);
7572}
7573
7574
7575/**
7576 * Sets a general-protection (#GP) exception as pending-for-injection into the
7577 * VM.
7578 *
7579 * @param pVCpu Pointer to the VMCPU.
7580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7581 * out-of-sync. Make sure to update the required fields
7582 * before using them.
7583 * @param u32ErrorCode The error code associated with the #GP.
7584 */
7585DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7586{
7587 NOREF(pMixedCtx);
7588 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7589 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7590 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7591 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7592}
7593
7594
7595/**
7596 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7597 *
7598 * @param pVCpu Pointer to the VMCPU.
7599 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7600 * out-of-sync. Make sure to update the required fields
7601 * before using them.
7602 * @param uVector The software interrupt vector number.
7603 * @param cbInstr The value of RIP that is to be pushed on the guest
7604 * stack.
7605 */
7606DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7607{
7608 NOREF(pMixedCtx);
7609 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7610 if ( uVector == X86_XCPT_BP
7611 || uVector == X86_XCPT_OF)
7612 {
7613 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7614 }
7615 else
7616 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7617 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7618}
7619
7620
7621/**
7622 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7623 * stack.
7624 *
7625 * @returns VBox status code (information status code included).
7626 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7627 * @param pVM Pointer to the VM.
7628 * @param pMixedCtx Pointer to the guest-CPU context.
7629 * @param uValue The value to push to the guest stack.
7630 */
7631DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7632{
7633 /*
7634 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7635 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7636 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7637 */
7638 if (pMixedCtx->sp == 1)
7639 return VINF_EM_RESET;
7640 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7641 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7642 AssertRCReturn(rc, rc);
7643 return rc;
7644}
7645
7646
7647/**
7648 * Injects an event into the guest upon VM-entry by updating the relevant fields
7649 * in the VM-entry area in the VMCS.
7650 *
7651 * @returns VBox status code (informational error codes included).
7652 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7653 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7654 *
7655 * @param pVCpu Pointer to the VMCPU.
7656 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7657 * be out-of-sync. Make sure to update the required
7658 * fields before using them.
7659 * @param u64IntInfo The VM-entry interruption-information field.
7660 * @param cbInstr The VM-entry instruction length in bytes (for
7661 * software interrupts, exceptions and privileged
7662 * software exceptions).
7663 * @param u32ErrCode The VM-entry exception error code.
7664 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7665 * @param puIntrState Pointer to the current guest interruptibility-state.
7666 * This interruptibility-state will be updated if
7667 * necessary. This cannot not be NULL.
7668 *
7669 * @remarks Requires CR0!
7670 * @remarks No-long-jump zone!!!
7671 */
7672static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7673 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7674{
7675 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7676 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7677 Assert(puIntrState);
7678 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7679
7680 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7681 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7682
7683#ifdef VBOX_STRICT
7684 /* Validate the error-code-valid bit for hardware exceptions. */
7685 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7686 {
7687 switch (uVector)
7688 {
7689 case X86_XCPT_PF:
7690 case X86_XCPT_DF:
7691 case X86_XCPT_TS:
7692 case X86_XCPT_NP:
7693 case X86_XCPT_SS:
7694 case X86_XCPT_GP:
7695 case X86_XCPT_AC:
7696 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7697 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7698 /* fallthru */
7699 default:
7700 break;
7701 }
7702 }
7703#endif
7704
7705 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7706 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7707 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7708
7709 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7710
7711 /* We require CR0 to check if the guest is in real-mode. */
7712 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7713 AssertRCReturn(rc, rc);
7714
7715 /*
7716 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7717 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7718 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7719 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7720 */
7721 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7722 {
7723 PVM pVM = pVCpu->CTX_SUFF(pVM);
7724 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7725 {
7726 Assert(PDMVmmDevHeapIsEnabled(pVM));
7727 Assert(pVM->hm.s.vmx.pRealModeTSS);
7728
7729 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7730 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7731 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7732 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7733 AssertRCReturn(rc, rc);
7734 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7735
7736 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7737 const size_t cbIdtEntry = sizeof(X86IDTR16);
7738 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7739 {
7740 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7741 if (uVector == X86_XCPT_DF)
7742 return VINF_EM_RESET;
7743 else if (uVector == X86_XCPT_GP)
7744 {
7745 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7746 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7747 }
7748
7749 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7750 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7751 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7752 }
7753
7754 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7755 uint16_t uGuestIp = pMixedCtx->ip;
7756 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7757 {
7758 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7759 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7760 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7761 }
7762 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7763 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7764
7765 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7766 X86IDTR16 IdtEntry;
7767 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7768 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7769 AssertRCReturn(rc, rc);
7770
7771 /* Construct the stack frame for the interrupt/exception handler. */
7772 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7773 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7774 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7775 AssertRCReturn(rc, rc);
7776
7777 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7778 if (rc == VINF_SUCCESS)
7779 {
7780 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7781 pMixedCtx->rip = IdtEntry.offSel;
7782 pMixedCtx->cs.Sel = IdtEntry.uSel;
7783 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7784 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7785 && uVector == X86_XCPT_PF)
7786 {
7787 pMixedCtx->cr2 = GCPtrFaultAddress;
7788 }
7789
7790 /* If any other guest-state bits are changed here, make sure to update
7791 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7792 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7793 | HM_CHANGED_GUEST_RIP
7794 | HM_CHANGED_GUEST_RFLAGS
7795 | HM_CHANGED_GUEST_RSP);
7796
7797 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7798 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7799 {
7800 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7801 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7802 Log4(("Clearing inhibition due to STI.\n"));
7803 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7804 }
7805 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7806
7807 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7808 it, if we are returning to ring-3 before executing guest code. */
7809 pVCpu->hm.s.Event.fPending = false;
7810 }
7811 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7812 return rc;
7813 }
7814 else
7815 {
7816 /*
7817 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7818 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7819 */
7820 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7821 }
7822 }
7823
7824 /* Validate. */
7825 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7826 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7827 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7828
7829 /* Inject. */
7830 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7831 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7832 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7833 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7834
7835 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7836 && uVector == X86_XCPT_PF)
7837 {
7838 pMixedCtx->cr2 = GCPtrFaultAddress;
7839 }
7840
7841 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7842 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7843
7844 AssertRCReturn(rc, rc);
7845 return rc;
7846}
7847
7848
7849/**
7850 * Clears the interrupt-window exiting control in the VMCS and if necessary
7851 * clears the current event in the VMCS as well.
7852 *
7853 * @returns VBox status code.
7854 * @param pVCpu Pointer to the VMCPU.
7855 *
7856 * @remarks Use this function only to clear events that have not yet been
7857 * delivered to the guest but are injected in the VMCS!
7858 * @remarks No-long-jump zone!!!
7859 */
7860static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7861{
7862 int rc;
7863 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7864
7865 /* Clear interrupt-window exiting control. */
7866 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7867 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7868
7869 if (!pVCpu->hm.s.Event.fPending)
7870 return;
7871
7872#ifdef VBOX_STRICT
7873 uint32_t u32EntryInfo;
7874 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7875 AssertRC(rc);
7876 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7877#endif
7878
7879 /* Clear the entry-interruption field (including the valid bit). */
7880 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7881 AssertRC(rc);
7882
7883 /* Clear the pending debug exception field. */
7884 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7885 AssertRC(rc);
7886
7887 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7888 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7889}
7890
7891
7892/**
7893 * Enters the VT-x session.
7894 *
7895 * @returns VBox status code.
7896 * @param pVM Pointer to the VM.
7897 * @param pVCpu Pointer to the VMCPU.
7898 * @param pCpu Pointer to the CPU info struct.
7899 */
7900VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7901{
7902 AssertPtr(pVM);
7903 AssertPtr(pVCpu);
7904 Assert(pVM->hm.s.vmx.fSupported);
7905 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7906 NOREF(pCpu); NOREF(pVM);
7907
7908 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7909 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7910
7911#ifdef VBOX_STRICT
7912 /* Make sure we're in VMX root mode. */
7913 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7914 if (!(u32HostCR4 & X86_CR4_VMXE))
7915 {
7916 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7917 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7918 }
7919#endif
7920
7921 /*
7922 * Load the VCPU's VMCS as the current (and active) one.
7923 */
7924 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7925 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7926 if (RT_FAILURE(rc))
7927 return rc;
7928
7929 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7930 pVCpu->hm.s.fLeaveDone = false;
7931 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7932
7933 return VINF_SUCCESS;
7934}
7935
7936
7937/**
7938 * The thread-context callback (only on platforms which support it).
7939 *
7940 * @param enmEvent The thread-context event.
7941 * @param pVCpu Pointer to the VMCPU.
7942 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7943 * @thread EMT(pVCpu)
7944 */
7945VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7946{
7947 NOREF(fGlobalInit);
7948
7949 switch (enmEvent)
7950 {
7951 case RTTHREADCTXEVENT_PREEMPTING:
7952 {
7953 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7954 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7955 VMCPU_ASSERT_EMT(pVCpu);
7956
7957 PVM pVM = pVCpu->CTX_SUFF(pVM);
7958 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7959
7960 /* No longjmps (logger flushes, locks) in this fragile context. */
7961 VMMRZCallRing3Disable(pVCpu);
7962 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7963
7964 /*
7965 * Restore host-state (FPU, debug etc.)
7966 */
7967 if (!pVCpu->hm.s.fLeaveDone)
7968 {
7969 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7970 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7971 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7972 pVCpu->hm.s.fLeaveDone = true;
7973 }
7974
7975 /* Leave HM context, takes care of local init (term). */
7976 int rc = HMR0LeaveCpu(pVCpu);
7977 AssertRC(rc); NOREF(rc);
7978
7979 /* Restore longjmp state. */
7980 VMMRZCallRing3Enable(pVCpu);
7981 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7982 break;
7983 }
7984
7985 case RTTHREADCTXEVENT_RESUMED:
7986 {
7987 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7988 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7989 VMCPU_ASSERT_EMT(pVCpu);
7990
7991 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7992 VMMRZCallRing3Disable(pVCpu);
7993 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7994
7995 /* Initialize the bare minimum state required for HM. This takes care of
7996 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7997 int rc = HMR0EnterCpu(pVCpu);
7998 AssertRC(rc);
7999 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8000
8001 /* Load the active VMCS as the current one. */
8002 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8003 {
8004 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8005 AssertRC(rc); NOREF(rc);
8006 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8007 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8008 }
8009 pVCpu->hm.s.fLeaveDone = false;
8010
8011 /* Restore longjmp state. */
8012 VMMRZCallRing3Enable(pVCpu);
8013 break;
8014 }
8015
8016 default:
8017 break;
8018 }
8019}
8020
8021
8022/**
8023 * Saves the host state in the VMCS host-state.
8024 * Sets up the VM-exit MSR-load area.
8025 *
8026 * The CPU state will be loaded from these fields on every successful VM-exit.
8027 *
8028 * @returns VBox status code.
8029 * @param pVM Pointer to the VM.
8030 * @param pVCpu Pointer to the VMCPU.
8031 *
8032 * @remarks No-long-jump zone!!!
8033 */
8034static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8035{
8036 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8037
8038 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8039 return VINF_SUCCESS;
8040
8041 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8042 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8043
8044 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8045 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8046
8047 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8049
8050 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8051 return rc;
8052}
8053
8054
8055/**
8056 * Saves the host state in the VMCS host-state.
8057 *
8058 * @returns VBox status code.
8059 * @param pVM Pointer to the VM.
8060 * @param pVCpu Pointer to the VMCPU.
8061 *
8062 * @remarks No-long-jump zone!!!
8063 */
8064VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8065{
8066 AssertPtr(pVM);
8067 AssertPtr(pVCpu);
8068
8069 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8070
8071 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8072 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8074 return hmR0VmxSaveHostState(pVM, pVCpu);
8075}
8076
8077
8078/**
8079 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8080 * loaded from these fields on every successful VM-entry.
8081 *
8082 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8083 * Sets up the VM-entry controls.
8084 * Sets up the appropriate VMX non-root function to execute guest code based on
8085 * the guest CPU mode.
8086 *
8087 * @returns VBox status code.
8088 * @param pVM Pointer to the VM.
8089 * @param pVCpu Pointer to the VMCPU.
8090 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8091 * out-of-sync. Make sure to update the required fields
8092 * before using them.
8093 *
8094 * @remarks No-long-jump zone!!!
8095 */
8096static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8097{
8098 AssertPtr(pVM);
8099 AssertPtr(pVCpu);
8100 AssertPtr(pMixedCtx);
8101 HMVMX_ASSERT_PREEMPT_SAFE();
8102
8103#ifdef LOG_ENABLED
8104 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8105 * probably not initialized yet? Anyway this will do for now.
8106 *
8107 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8108 * interface and disable ring-3 calls when thread-context hooks are not
8109 * available. */
8110 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8111 VMMR0LogFlushDisable(pVCpu);
8112#endif
8113
8114 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8115
8116 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8117
8118 /* Determine real-on-v86 mode. */
8119 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8120 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8121 && CPUMIsGuestInRealModeEx(pMixedCtx))
8122 {
8123 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8124 }
8125
8126 /*
8127 * Load the guest-state into the VMCS.
8128 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8129 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8130 */
8131 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8132 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8133
8134 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8135 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8136 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8137
8138 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8139 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8140 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8141
8142 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8143 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8144
8145 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8146 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8147
8148 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8149 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8150 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8151
8152 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8153 determine we don't have to swap EFER after all. */
8154 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8155 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8156
8157 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8158 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8159
8160 /*
8161 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8162 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8163 */
8164 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8165 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8166
8167 /* Clear any unused and reserved bits. */
8168 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8169
8170#ifdef LOG_ENABLED
8171 /* Only reenable log-flushing if the caller has it enabled. */
8172 if (!fCallerDisabledLogFlush)
8173 VMMR0LogFlushEnable(pVCpu);
8174#endif
8175
8176 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8177 return rc;
8178}
8179
8180
8181/**
8182 * Loads the state shared between the host and guest into the VMCS.
8183 *
8184 * @param pVM Pointer to the VM.
8185 * @param pVCpu Pointer to the VMCPU.
8186 * @param pCtx Pointer to the guest-CPU context.
8187 *
8188 * @remarks No-long-jump zone!!!
8189 */
8190static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8191{
8192 NOREF(pVM);
8193
8194 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8196
8197 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8198 {
8199 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8200 AssertRC(rc);
8201 }
8202
8203 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8204 {
8205 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8206 AssertRC(rc);
8207
8208 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8209 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8210 {
8211 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8212 AssertRC(rc);
8213 }
8214 }
8215
8216 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8217 {
8218#if HC_ARCH_BITS == 64
8219 if (pVM->hm.s.fAllow64BitGuests)
8220 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8221#endif
8222 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8223 }
8224
8225 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8226 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8227}
8228
8229
8230/**
8231 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8232 *
8233 * @param pVM Pointer to the VM.
8234 * @param pVCpu Pointer to the VMCPU.
8235 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8236 * out-of-sync. Make sure to update the required fields
8237 * before using them.
8238 */
8239DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8240{
8241 HMVMX_ASSERT_PREEMPT_SAFE();
8242
8243 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8244#ifdef HMVMX_SYNC_FULL_GUEST_STATE
8245 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8246#endif
8247
8248 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8249 {
8250 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8251 AssertRC(rc);
8252 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8253 }
8254 else if (HMCPU_CF_VALUE(pVCpu))
8255 {
8256 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8257 AssertRC(rc);
8258 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8259 }
8260
8261 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8262 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8263 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8264 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8265}
8266
8267
8268/**
8269 * Does the preparations before executing guest code in VT-x.
8270 *
8271 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8272 * recompiler. We must be cautious what we do here regarding committing
8273 * guest-state information into the VMCS assuming we assuredly execute the
8274 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8275 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8276 * so that the recompiler can (and should) use them when it resumes guest
8277 * execution. Otherwise such operations must be done when we can no longer
8278 * exit to ring-3.
8279 *
8280 * @returns Strict VBox status code.
8281 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8282 * have been disabled.
8283 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8284 * double-fault into the guest.
8285 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8286 *
8287 * @param pVM Pointer to the VM.
8288 * @param pVCpu Pointer to the VMCPU.
8289 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8290 * out-of-sync. Make sure to update the required fields
8291 * before using them.
8292 * @param pVmxTransient Pointer to the VMX transient structure.
8293 */
8294static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8295{
8296 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8297
8298#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8299 PGMRZDynMapFlushAutoSet(pVCpu);
8300#endif
8301
8302 /* Check force flag actions that might require us to go back to ring-3. */
8303 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8304 if (rc != VINF_SUCCESS)
8305 return rc;
8306
8307#ifndef IEM_VERIFICATION_MODE_FULL
8308 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8309 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8310 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8311 {
8312 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8313 RTGCPHYS GCPhysApicBase;
8314 GCPhysApicBase = pMixedCtx->msrApicBase;
8315 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8316
8317 /* Unalias any existing mapping. */
8318 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8319 AssertRCReturn(rc, rc);
8320
8321 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8322 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8323 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8324 AssertRCReturn(rc, rc);
8325
8326 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8327 }
8328#endif /* !IEM_VERIFICATION_MODE_FULL */
8329
8330 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8331 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8332
8333 /*
8334 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8335 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8336 */
8337 if (TRPMHasTrap(pVCpu))
8338 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8339 else if (!pVCpu->hm.s.Event.fPending)
8340 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8341
8342 /*
8343 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8344 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8345 */
8346 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8347 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8348 {
8349 Assert(rc == VINF_EM_RESET);
8350 return rc;
8351 }
8352
8353 /*
8354 * No longjmps to ring-3 from this point on!!!
8355 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8356 * This also disables flushing of the R0-logger instance (if any).
8357 */
8358 VMMRZCallRing3Disable(pVCpu);
8359
8360 /*
8361 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8362 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8363 *
8364 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8365 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8366 *
8367 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8368 * executing guest code.
8369 */
8370 pVmxTransient->uEflags = ASMIntDisableFlags();
8371 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8372 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8373 {
8374 hmR0VmxClearEventVmcs(pVCpu);
8375 ASMSetFlags(pVmxTransient->uEflags);
8376 VMMRZCallRing3Enable(pVCpu);
8377 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8378 return VINF_EM_RAW_TO_R3;
8379 }
8380
8381 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8382 {
8383 hmR0VmxClearEventVmcs(pVCpu);
8384 ASMSetFlags(pVmxTransient->uEflags);
8385 VMMRZCallRing3Enable(pVCpu);
8386 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8387 return VINF_EM_RAW_INTERRUPT;
8388 }
8389
8390 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8391 pVCpu->hm.s.Event.fPending = false;
8392
8393 return VINF_SUCCESS;
8394}
8395
8396
8397/**
8398 * Prepares to run guest code in VT-x and we've committed to doing so. This
8399 * means there is no backing out to ring-3 or anywhere else at this
8400 * point.
8401 *
8402 * @param pVM Pointer to the VM.
8403 * @param pVCpu Pointer to the VMCPU.
8404 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8405 * out-of-sync. Make sure to update the required fields
8406 * before using them.
8407 * @param pVmxTransient Pointer to the VMX transient structure.
8408 *
8409 * @remarks Called with preemption disabled.
8410 * @remarks No-long-jump zone!!!
8411 */
8412static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8413{
8414 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8415 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8416 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8417
8418 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8419 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8420
8421 /*
8422 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8423 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8424 * Reload only the necessary state, the assertion will catch if other parts of the code
8425 * change.
8426 */
8427 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8428 {
8429 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8430 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8431 }
8432
8433#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8434 if (!CPUMIsGuestFPUStateActive(pVCpu))
8435 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8436 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8437#endif
8438
8439 if ( pVCpu->hm.s.fUseGuestFpu
8440 && !CPUMIsGuestFPUStateActive(pVCpu))
8441 {
8442 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8443 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8444 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8445 }
8446
8447 /*
8448 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8449 */
8450 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8451 && pVCpu->hm.s.vmx.cMsrs > 0)
8452 {
8453 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8454 }
8455
8456 /*
8457 * Load the host state bits as we may've been preempted (only happens when
8458 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8459 */
8460 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8461 {
8462 /* This ASSUMES that pfnStartVM has been set up already. */
8463 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8464 AssertRC(rc);
8465 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8466 }
8467 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8468
8469 /*
8470 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8471 */
8472 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8473 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8474 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8475
8476 /* Store status of the shared guest-host state at the time of VM-entry. */
8477#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8478 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8479 {
8480 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8481 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8482 }
8483 else
8484#endif
8485 {
8486 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8487 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8488 }
8489 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8490
8491 /*
8492 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8493 */
8494 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8495 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8496
8497 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8498 RTCPUID idCurrentCpu = pCpu->idCpu;
8499 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8500 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8501 {
8502 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8503 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8504 }
8505
8506 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8507 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8508 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8509 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8510
8511 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8512
8513 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8514 to start executing. */
8515
8516 /*
8517 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8518 */
8519 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8520 {
8521 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8522 {
8523 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8524 AssertRC(rc2);
8525 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8526 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8527 true /* fUpdateHostMsr */);
8528 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8529 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8530 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8531 }
8532 else
8533 {
8534 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8535 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8536 }
8537 }
8538
8539#ifdef VBOX_STRICT
8540 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8541 hmR0VmxCheckHostEferMsr(pVCpu);
8542 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8543#endif
8544#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8545 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8546 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8547 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8548#endif
8549}
8550
8551
8552/**
8553 * Performs some essential restoration of state after running guest code in
8554 * VT-x.
8555 *
8556 * @param pVM Pointer to the VM.
8557 * @param pVCpu Pointer to the VMCPU.
8558 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8559 * out-of-sync. Make sure to update the required fields
8560 * before using them.
8561 * @param pVmxTransient Pointer to the VMX transient structure.
8562 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8563 *
8564 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8565 *
8566 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8567 * unconditionally when it is safe to do so.
8568 */
8569static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8570{
8571 NOREF(pVM);
8572
8573 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8574
8575 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8576 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8577 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8578 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8579 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8580
8581 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8582 {
8583 /** @todo Find a way to fix hardcoding a guestimate. */
8584 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8585 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8586 }
8587
8588 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8589 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8590 Assert(!(ASMGetFlags() & X86_EFL_IF));
8591 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8592
8593#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8594 if (CPUMIsGuestFPUStateActive(pVCpu))
8595 {
8596 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8597 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8598 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8599 }
8600#endif
8601
8602#if HC_ARCH_BITS == 64
8603 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8604#endif
8605 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8606#ifdef VBOX_STRICT
8607 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8608#endif
8609 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8610 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8611
8612 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8613 uint32_t uExitReason;
8614 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8615 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8616 AssertRC(rc);
8617 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8618 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8619
8620 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8621 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8622 {
8623 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8624 pVmxTransient->fVMEntryFailed));
8625 return;
8626 }
8627
8628 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8629 {
8630 /* Update the guest interruptibility-state from the VMCS. */
8631 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8632#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8633 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8634 AssertRC(rc);
8635#endif
8636 /*
8637 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8638 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8639 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8640 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8641 */
8642 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8643 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8644 {
8645 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8646 AssertRC(rc);
8647 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8648 }
8649 }
8650}
8651
8652
8653/**
8654 * Runs the guest code using VT-x the normal way.
8655 *
8656 * @returns VBox status code.
8657 * @param pVM Pointer to the VM.
8658 * @param pVCpu Pointer to the VMCPU.
8659 * @param pCtx Pointer to the guest-CPU context.
8660 *
8661 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8662 */
8663static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8664{
8665 VMXTRANSIENT VmxTransient;
8666 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8667 int rc = VERR_INTERNAL_ERROR_5;
8668 uint32_t cLoops = 0;
8669
8670 for (;; cLoops++)
8671 {
8672 Assert(!HMR0SuspendPending());
8673 HMVMX_ASSERT_CPU_SAFE();
8674
8675 /* Preparatory work for running guest code, this may force us to return
8676 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8677 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8678 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8679 if (rc != VINF_SUCCESS)
8680 break;
8681
8682 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8683 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8684 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8685
8686 /* Restore any residual host-state and save any bits shared between host
8687 and guest into the guest-CPU state. Re-enables interrupts! */
8688 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8689
8690 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8691 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8692 {
8693 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8694 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8695 return rc;
8696 }
8697
8698 /* Handle the VM-exit. */
8699 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8701 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8702 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8703 HMVMX_START_EXIT_DISPATCH_PROF();
8704#ifdef HMVMX_USE_FUNCTION_TABLE
8705 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8706#else
8707 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8708#endif
8709 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8710 if (rc != VINF_SUCCESS)
8711 break;
8712 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8713 {
8714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8715 rc = VINF_EM_RAW_INTERRUPT;
8716 break;
8717 }
8718 }
8719
8720 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8721 return rc;
8722}
8723
8724
8725/**
8726 * Single steps guest code using VT-x.
8727 *
8728 * @returns VBox status code.
8729 * @param pVM Pointer to the VM.
8730 * @param pVCpu Pointer to the VMCPU.
8731 * @param pCtx Pointer to the guest-CPU context.
8732 *
8733 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8734 */
8735static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8736{
8737 VMXTRANSIENT VmxTransient;
8738 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8739 int rc = VERR_INTERNAL_ERROR_5;
8740 uint32_t cLoops = 0;
8741 uint16_t uCsStart = pCtx->cs.Sel;
8742 uint64_t uRipStart = pCtx->rip;
8743
8744 for (;; cLoops++)
8745 {
8746 Assert(!HMR0SuspendPending());
8747 HMVMX_ASSERT_CPU_SAFE();
8748
8749 /* Preparatory work for running guest code, this may force us to return
8750 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8751 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8752 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8753 if (rc != VINF_SUCCESS)
8754 break;
8755
8756 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8757 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8758 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8759
8760 /* Restore any residual host-state and save any bits shared between host
8761 and guest into the guest-CPU state. Re-enables interrupts! */
8762 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8763
8764 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8765 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8766 {
8767 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8768 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8769 return rc;
8770 }
8771
8772 /* Handle the VM-exit. */
8773 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8775 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8776 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8777 HMVMX_START_EXIT_DISPATCH_PROF();
8778#ifdef HMVMX_USE_FUNCTION_TABLE
8779 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8780#else
8781 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8782#endif
8783 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8784 if (rc != VINF_SUCCESS)
8785 break;
8786 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8787 {
8788 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8789 rc = VINF_EM_RAW_INTERRUPT;
8790 break;
8791 }
8792
8793 /*
8794 * Did the RIP change, if so, consider it a single step.
8795 * Otherwise, make sure one of the TFs gets set.
8796 */
8797 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8798 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8799 AssertRCReturn(rc2, rc2);
8800 if ( pCtx->rip != uRipStart
8801 || pCtx->cs.Sel != uCsStart)
8802 {
8803 rc = VINF_EM_DBG_STEPPED;
8804 break;
8805 }
8806 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8807 }
8808
8809 /*
8810 * Clear the X86_EFL_TF if necessary.
8811 */
8812 if (pVCpu->hm.s.fClearTrapFlag)
8813 {
8814 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8815 AssertRCReturn(rc2, rc2);
8816 pVCpu->hm.s.fClearTrapFlag = false;
8817 pCtx->eflags.Bits.u1TF = 0;
8818 }
8819 /** @todo there seems to be issues with the resume flag when the monitor trap
8820 * flag is pending without being used. Seen early in bios init when
8821 * accessing APIC page in protected mode. */
8822
8823 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8824 return rc;
8825}
8826
8827
8828/**
8829 * Runs the guest code using VT-x.
8830 *
8831 * @returns VBox status code.
8832 * @param pVM Pointer to the VM.
8833 * @param pVCpu Pointer to the VMCPU.
8834 * @param pCtx Pointer to the guest-CPU context.
8835 */
8836VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8837{
8838 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8839 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8840 HMVMX_ASSERT_PREEMPT_SAFE();
8841
8842 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8843
8844 int rc;
8845 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8846 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8847 else
8848 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8849
8850 if (rc == VERR_EM_INTERPRETER)
8851 rc = VINF_EM_RAW_EMULATE_INSTR;
8852 else if (rc == VINF_EM_RESET)
8853 rc = VINF_EM_TRIPLE_FAULT;
8854
8855 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8856 if (RT_FAILURE(rc2))
8857 {
8858 pVCpu->hm.s.u32HMError = rc;
8859 rc = rc2;
8860 }
8861 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8862 return rc;
8863}
8864
8865
8866#ifndef HMVMX_USE_FUNCTION_TABLE
8867DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8868{
8869#ifdef DEBUG_ramshankar
8870# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8871# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8872#endif
8873 int rc;
8874 switch (rcReason)
8875 {
8876 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8877 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8878 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8879 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8880 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8881 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8882 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8883 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8884 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8885 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8886 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8887 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8888 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8889 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8890 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8891 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8892 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8893 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8894 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8895 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8896 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8897 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8898 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8899 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8900 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8901 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8902 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8903 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8904 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8905 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8906 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8907 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8908 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8909
8910 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8911 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8912 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8913 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8914 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8915 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8916 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8917 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8918 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8919
8920 case VMX_EXIT_VMCALL:
8921 case VMX_EXIT_VMCLEAR:
8922 case VMX_EXIT_VMLAUNCH:
8923 case VMX_EXIT_VMPTRLD:
8924 case VMX_EXIT_VMPTRST:
8925 case VMX_EXIT_VMREAD:
8926 case VMX_EXIT_VMRESUME:
8927 case VMX_EXIT_VMWRITE:
8928 case VMX_EXIT_VMXOFF:
8929 case VMX_EXIT_VMXON:
8930 case VMX_EXIT_INVEPT:
8931 case VMX_EXIT_INVVPID:
8932 case VMX_EXIT_VMFUNC:
8933 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8934 break;
8935 default:
8936 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8937 break;
8938 }
8939 return rc;
8940}
8941#endif
8942
8943#ifdef DEBUG
8944/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8945# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8946 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8947
8948# define HMVMX_ASSERT_PREEMPT_CPUID() \
8949 do \
8950 { \
8951 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8952 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8953 } while (0)
8954
8955# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8956 do { \
8957 AssertPtr(pVCpu); \
8958 AssertPtr(pMixedCtx); \
8959 AssertPtr(pVmxTransient); \
8960 Assert(pVmxTransient->fVMEntryFailed == false); \
8961 Assert(ASMIntAreEnabled()); \
8962 HMVMX_ASSERT_PREEMPT_SAFE(); \
8963 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8964 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8965 HMVMX_ASSERT_PREEMPT_SAFE(); \
8966 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8967 HMVMX_ASSERT_PREEMPT_CPUID(); \
8968 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8969 } while (0)
8970
8971# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8972 do { \
8973 Log4Func(("\n")); \
8974 } while (0)
8975#else /* Release builds */
8976# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8977 do { \
8978 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8979 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8980 } while (0)
8981# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8982#endif
8983
8984
8985/**
8986 * Advances the guest RIP after reading it from the VMCS.
8987 *
8988 * @returns VBox status code.
8989 * @param pVCpu Pointer to the VMCPU.
8990 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8991 * out-of-sync. Make sure to update the required fields
8992 * before using them.
8993 * @param pVmxTransient Pointer to the VMX transient structure.
8994 *
8995 * @remarks No-long-jump zone!!!
8996 */
8997DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8998{
8999 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9000 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9001 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9002 AssertRCReturn(rc, rc);
9003
9004 pMixedCtx->rip += pVmxTransient->cbInstr;
9005 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9006
9007 /*
9008 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9009 * pending debug exception field as it takes care of priority of events.
9010 *
9011 * See Intel spec. 32.2.1 "Debug Exceptions".
9012 */
9013 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9014
9015 return rc;
9016}
9017
9018
9019/**
9020 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9021 * and update error record fields accordingly.
9022 *
9023 * @return VMX_IGS_* return codes.
9024 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9025 * wrong with the guest state.
9026 *
9027 * @param pVM Pointer to the VM.
9028 * @param pVCpu Pointer to the VMCPU.
9029 * @param pCtx Pointer to the guest-CPU state.
9030 *
9031 * @remarks This function assumes our cache of the VMCS controls
9032 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9033 */
9034static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9035{
9036#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9037#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9038 uError = (err); \
9039 break; \
9040 } else do { } while (0)
9041
9042 int rc;
9043 uint32_t uError = VMX_IGS_ERROR;
9044 uint32_t u32Val;
9045 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9046
9047 do
9048 {
9049 /*
9050 * CR0.
9051 */
9052 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9053 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9054 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9055 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9056 if (fUnrestrictedGuest)
9057 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9058
9059 uint32_t u32GuestCR0;
9060 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9061 AssertRCBreak(rc);
9062 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9063 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9064 if ( !fUnrestrictedGuest
9065 && (u32GuestCR0 & X86_CR0_PG)
9066 && !(u32GuestCR0 & X86_CR0_PE))
9067 {
9068 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9069 }
9070
9071 /*
9072 * CR4.
9073 */
9074 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9075 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9076
9077 uint32_t u32GuestCR4;
9078 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9079 AssertRCBreak(rc);
9080 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9081 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9082
9083 /*
9084 * IA32_DEBUGCTL MSR.
9085 */
9086 uint64_t u64Val;
9087 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9088 AssertRCBreak(rc);
9089 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9090 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9091 {
9092 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9093 }
9094 uint64_t u64DebugCtlMsr = u64Val;
9095
9096#ifdef VBOX_STRICT
9097 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9098 AssertRCBreak(rc);
9099 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9100#endif
9101 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9102
9103 /*
9104 * RIP and RFLAGS.
9105 */
9106 uint32_t u32Eflags;
9107#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9108 if (HMVMX_IS_64BIT_HOST_MODE())
9109 {
9110 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9111 AssertRCBreak(rc);
9112 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9113 if ( !fLongModeGuest
9114 || !pCtx->cs.Attr.n.u1Long)
9115 {
9116 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9117 }
9118 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9119 * must be identical if the "IA-32e mode guest" VM-entry
9120 * control is 1 and CS.L is 1. No check applies if the
9121 * CPU supports 64 linear-address bits. */
9122
9123 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9124 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9125 AssertRCBreak(rc);
9126 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9127 VMX_IGS_RFLAGS_RESERVED);
9128 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9129 u32Eflags = u64Val;
9130 }
9131 else
9132#endif
9133 {
9134 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9135 AssertRCBreak(rc);
9136 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9137 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9138 }
9139
9140 if ( fLongModeGuest
9141 || ( fUnrestrictedGuest
9142 && !(u32GuestCR0 & X86_CR0_PE)))
9143 {
9144 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9145 }
9146
9147 uint32_t u32EntryInfo;
9148 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9149 AssertRCBreak(rc);
9150 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9151 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9152 {
9153 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9154 }
9155
9156 /*
9157 * 64-bit checks.
9158 */
9159#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9160 if (HMVMX_IS_64BIT_HOST_MODE())
9161 {
9162 if ( fLongModeGuest
9163 && !fUnrestrictedGuest)
9164 {
9165 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9166 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9167 }
9168
9169 if ( !fLongModeGuest
9170 && (u32GuestCR4 & X86_CR4_PCIDE))
9171 {
9172 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9173 }
9174
9175 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9176 * 51:32 beyond the processor's physical-address width are 0. */
9177
9178 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9179 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9180 {
9181 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9182 }
9183
9184 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9185 AssertRCBreak(rc);
9186 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9187
9188 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9189 AssertRCBreak(rc);
9190 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9191 }
9192#endif
9193
9194 /*
9195 * PERF_GLOBAL MSR.
9196 */
9197 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9198 {
9199 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9200 AssertRCBreak(rc);
9201 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9202 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9203 }
9204
9205 /*
9206 * PAT MSR.
9207 */
9208 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9209 {
9210 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9211 AssertRCBreak(rc);
9212 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9213 for (unsigned i = 0; i < 8; i++)
9214 {
9215 uint8_t u8Val = (u64Val & 0x7);
9216 if ( u8Val != 0 /* UC */
9217 || u8Val != 1 /* WC */
9218 || u8Val != 4 /* WT */
9219 || u8Val != 5 /* WP */
9220 || u8Val != 6 /* WB */
9221 || u8Val != 7 /* UC- */)
9222 {
9223 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9224 }
9225 u64Val >>= 3;
9226 }
9227 }
9228
9229 /*
9230 * EFER MSR.
9231 */
9232 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9233 {
9234 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9235 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9236 AssertRCBreak(rc);
9237 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9238 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9239 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9240 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9241 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9242 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9243 VMX_IGS_EFER_LMA_PG_MISMATCH);
9244 }
9245
9246 /*
9247 * Segment registers.
9248 */
9249 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9250 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9251 if (!(u32Eflags & X86_EFL_VM))
9252 {
9253 /* CS */
9254 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9255 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9256 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9257 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9258 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9259 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9260 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9261 /* CS cannot be loaded with NULL in protected mode. */
9262 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9263 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9264 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9265 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9266 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9267 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9268 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9269 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9270 else
9271 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9272
9273 /* SS */
9274 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9275 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9276 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9277 if ( !(pCtx->cr0 & X86_CR0_PE)
9278 || pCtx->cs.Attr.n.u4Type == 3)
9279 {
9280 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9281 }
9282 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9283 {
9284 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9285 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9286 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9287 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9288 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9289 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9290 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9291 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9292 }
9293
9294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9295 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9296 {
9297 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9298 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9299 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9300 || pCtx->ds.Attr.n.u4Type > 11
9301 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9302 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9303 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9304 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9305 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9306 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9307 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9308 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9309 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9310 }
9311 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9312 {
9313 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9314 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9315 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9316 || pCtx->es.Attr.n.u4Type > 11
9317 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9318 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9319 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9320 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9321 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9322 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9323 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9324 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9325 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9326 }
9327 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9328 {
9329 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9330 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9331 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9332 || pCtx->fs.Attr.n.u4Type > 11
9333 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9334 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9335 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9336 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9337 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9338 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9339 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9340 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9341 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9342 }
9343 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9344 {
9345 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9346 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9347 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9348 || pCtx->gs.Attr.n.u4Type > 11
9349 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9350 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9351 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9352 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9353 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9354 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9355 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9356 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9357 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9358 }
9359 /* 64-bit capable CPUs. */
9360#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9361 if (HMVMX_IS_64BIT_HOST_MODE())
9362 {
9363 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9364 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9365 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9366 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9367 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9368 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9369 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9370 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9371 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9372 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9373 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9374 }
9375#endif
9376 }
9377 else
9378 {
9379 /* V86 mode checks. */
9380 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9381 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9382 {
9383 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9384 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9385 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9386 }
9387 else
9388 {
9389 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9390 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9391 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9392 }
9393
9394 /* CS */
9395 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9396 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9397 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9398 /* SS */
9399 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9400 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9401 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9402 /* DS */
9403 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9404 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9405 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9406 /* ES */
9407 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9408 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9409 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9410 /* FS */
9411 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9412 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9413 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9414 /* GS */
9415 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9416 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9417 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9418 /* 64-bit capable CPUs. */
9419#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9420 if (HMVMX_IS_64BIT_HOST_MODE())
9421 {
9422 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9423 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9424 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9425 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9426 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9427 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9428 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9429 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9430 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9431 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9432 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9433 }
9434#endif
9435 }
9436
9437 /*
9438 * TR.
9439 */
9440 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9441 /* 64-bit capable CPUs. */
9442#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9443 if (HMVMX_IS_64BIT_HOST_MODE())
9444 {
9445 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9446 }
9447#endif
9448 if (fLongModeGuest)
9449 {
9450 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9451 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9452 }
9453 else
9454 {
9455 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9456 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9457 VMX_IGS_TR_ATTR_TYPE_INVALID);
9458 }
9459 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9460 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9461 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9462 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9463 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9464 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9465 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9466 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9467
9468 /*
9469 * GDTR and IDTR.
9470 */
9471#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9472 if (HMVMX_IS_64BIT_HOST_MODE())
9473 {
9474 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9475 AssertRCBreak(rc);
9476 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9477
9478 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9479 AssertRCBreak(rc);
9480 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9481 }
9482#endif
9483
9484 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9485 AssertRCBreak(rc);
9486 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9487
9488 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9489 AssertRCBreak(rc);
9490 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9491
9492 /*
9493 * Guest Non-Register State.
9494 */
9495 /* Activity State. */
9496 uint32_t u32ActivityState;
9497 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9498 AssertRCBreak(rc);
9499 HMVMX_CHECK_BREAK( !u32ActivityState
9500 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9501 VMX_IGS_ACTIVITY_STATE_INVALID);
9502 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9503 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9504 uint32_t u32IntrState;
9505 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9506 AssertRCBreak(rc);
9507 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9508 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9509 {
9510 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9511 }
9512
9513 /** @todo Activity state and injecting interrupts. Left as a todo since we
9514 * currently don't use activity states but ACTIVE. */
9515
9516 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9517 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9518
9519 /* Guest interruptibility-state. */
9520 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9521 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9522 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9523 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9524 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9525 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9526 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9527 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9528 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9529 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9530 {
9531 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9532 {
9533 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9534 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9535 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9536 }
9537 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9538 {
9539 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9540 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9541 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9542 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9543 }
9544 }
9545 /** @todo Assumes the processor is not in SMM. */
9546 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9547 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9548 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9549 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9550 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9551 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9552 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9553 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9554 {
9555 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9556 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9557 }
9558
9559 /* Pending debug exceptions. */
9560 if (HMVMX_IS_64BIT_HOST_MODE())
9561 {
9562 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9563 AssertRCBreak(rc);
9564 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9565 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9566 u32Val = u64Val; /* For pending debug exceptions checks below. */
9567 }
9568 else
9569 {
9570 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9571 AssertRCBreak(rc);
9572 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9573 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9574 }
9575
9576 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9577 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9578 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9579 {
9580 if ( (u32Eflags & X86_EFL_TF)
9581 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9582 {
9583 /* Bit 14 is PendingDebug.BS. */
9584 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9585 }
9586 if ( !(u32Eflags & X86_EFL_TF)
9587 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9588 {
9589 /* Bit 14 is PendingDebug.BS. */
9590 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9591 }
9592 }
9593
9594 /* VMCS link pointer. */
9595 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9596 AssertRCBreak(rc);
9597 if (u64Val != UINT64_C(0xffffffffffffffff))
9598 {
9599 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9600 /** @todo Bits beyond the processor's physical-address width MBZ. */
9601 /** @todo 32-bit located in memory referenced by value of this field (as a
9602 * physical address) must contain the processor's VMCS revision ID. */
9603 /** @todo SMM checks. */
9604 }
9605
9606 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9607 * not using Nested Paging? */
9608 if ( pVM->hm.s.fNestedPaging
9609 && !fLongModeGuest
9610 && CPUMIsGuestInPAEModeEx(pCtx))
9611 {
9612 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9613 AssertRCBreak(rc);
9614 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9615
9616 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9617 AssertRCBreak(rc);
9618 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9619
9620 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9621 AssertRCBreak(rc);
9622 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9623
9624 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9625 AssertRCBreak(rc);
9626 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9627 }
9628
9629 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9630 if (uError == VMX_IGS_ERROR)
9631 uError = VMX_IGS_REASON_NOT_FOUND;
9632 } while (0);
9633
9634 pVCpu->hm.s.u32HMError = uError;
9635 return uError;
9636
9637#undef HMVMX_ERROR_BREAK
9638#undef HMVMX_CHECK_BREAK
9639}
9640
9641/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9642/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9643/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9644
9645/** @name VM-exit handlers.
9646 * @{
9647 */
9648
9649/**
9650 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9651 */
9652HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9653{
9654 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9656 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9657 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9658 return VINF_SUCCESS;
9659 return VINF_EM_RAW_INTERRUPT;
9660}
9661
9662
9663/**
9664 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9665 */
9666HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9667{
9668 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9669 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9670
9671 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9672 AssertRCReturn(rc, rc);
9673
9674 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9675 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9676 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9677 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9678
9679 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9680 {
9681 /*
9682 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9683 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9684 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9685 *
9686 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9687 */
9688 VMXDispatchHostNmi();
9689 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9690 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9691 return VINF_SUCCESS;
9692 }
9693
9694 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9695 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9696 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9697 {
9698 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9699 return VINF_SUCCESS;
9700 }
9701 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9702 {
9703 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9704 return rc;
9705 }
9706
9707 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9708 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9709 switch (uIntType)
9710 {
9711 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9712 Assert(uVector == X86_XCPT_DB);
9713 /* no break */
9714 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9715 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9716 /* no break */
9717 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9718 {
9719 switch (uVector)
9720 {
9721 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9722 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9723 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9724 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9725 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9726 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9727#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9728 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9729 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9730 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9731 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9732 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9733 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9734 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9735 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9736 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9737 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9738 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9739 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9740#endif
9741 default:
9742 {
9743 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9744 AssertRCReturn(rc, rc);
9745
9746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9747 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9748 {
9749 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9750 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9751 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9752
9753 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9754 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9755 AssertRCReturn(rc, rc);
9756 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9757 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9758 0 /* GCPtrFaultAddress */);
9759 AssertRCReturn(rc, rc);
9760 }
9761 else
9762 {
9763 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9764 pVCpu->hm.s.u32HMError = uVector;
9765 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9766 }
9767 break;
9768 }
9769 }
9770 break;
9771 }
9772
9773 default:
9774 {
9775 pVCpu->hm.s.u32HMError = uExitIntInfo;
9776 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9777 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9778 break;
9779 }
9780 }
9781 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9782 return rc;
9783}
9784
9785
9786/**
9787 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9788 */
9789HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790{
9791 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9792
9793 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9794 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9795
9796 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9797 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9798 return VINF_SUCCESS;
9799}
9800
9801
9802/**
9803 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9804 */
9805HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9806{
9807 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9808 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9809 HMVMX_RETURN_UNEXPECTED_EXIT();
9810}
9811
9812
9813/**
9814 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9815 */
9816HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9817{
9818 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9819 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9820 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9821}
9822
9823
9824/**
9825 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9826 */
9827HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9828{
9829 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9830 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9831 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9832}
9833
9834
9835/**
9836 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9837 */
9838HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9839{
9840 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9841 PVM pVM = pVCpu->CTX_SUFF(pVM);
9842 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9843 if (RT_LIKELY(rc == VINF_SUCCESS))
9844 {
9845 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9846 Assert(pVmxTransient->cbInstr == 2);
9847 }
9848 else
9849 {
9850 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9851 rc = VERR_EM_INTERPRETER;
9852 }
9853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9854 return rc;
9855}
9856
9857
9858/**
9859 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9860 */
9861HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9862{
9863 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9864 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9865 AssertRCReturn(rc, rc);
9866
9867 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9868 return VINF_EM_RAW_EMULATE_INSTR;
9869
9870 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9871 HMVMX_RETURN_UNEXPECTED_EXIT();
9872}
9873
9874
9875/**
9876 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9877 */
9878HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9879{
9880 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9881 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9882 AssertRCReturn(rc, rc);
9883
9884 PVM pVM = pVCpu->CTX_SUFF(pVM);
9885 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9886 if (RT_LIKELY(rc == VINF_SUCCESS))
9887 {
9888 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9889 Assert(pVmxTransient->cbInstr == 2);
9890 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9891 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9892 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9893 }
9894 else
9895 {
9896 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9897 rc = VERR_EM_INTERPRETER;
9898 }
9899 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9900 return rc;
9901}
9902
9903
9904/**
9905 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9906 */
9907HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9908{
9909 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9910 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9911 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9912 AssertRCReturn(rc, rc);
9913
9914 PVM pVM = pVCpu->CTX_SUFF(pVM);
9915 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9916 if (RT_LIKELY(rc == VINF_SUCCESS))
9917 {
9918 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9919 Assert(pVmxTransient->cbInstr == 3);
9920 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9921 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9922 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9923 }
9924 else
9925 {
9926 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9927 rc = VERR_EM_INTERPRETER;
9928 }
9929 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9930 return rc;
9931}
9932
9933
9934/**
9935 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9936 */
9937HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9938{
9939 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9940 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9941 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9942 AssertRCReturn(rc, rc);
9943
9944 PVM pVM = pVCpu->CTX_SUFF(pVM);
9945 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9946 if (RT_LIKELY(rc == VINF_SUCCESS))
9947 {
9948 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9949 Assert(pVmxTransient->cbInstr == 2);
9950 }
9951 else
9952 {
9953 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9954 rc = VERR_EM_INTERPRETER;
9955 }
9956 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9957 return rc;
9958}
9959
9960
9961/**
9962 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9963 */
9964HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9965{
9966 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9967 PVM pVM = pVCpu->CTX_SUFF(pVM);
9968 Assert(!pVM->hm.s.fNestedPaging);
9969
9970 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9971 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9972 AssertRCReturn(rc, rc);
9973
9974 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9975 rc = VBOXSTRICTRC_VAL(rc2);
9976 if (RT_LIKELY(rc == VINF_SUCCESS))
9977 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9978 else
9979 {
9980 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9981 pVmxTransient->uExitQualification, rc));
9982 }
9983 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9984 return rc;
9985}
9986
9987
9988/**
9989 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9990 */
9991HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9992{
9993 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9994 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9995 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9996 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9997 AssertRCReturn(rc, rc);
9998
9999 PVM pVM = pVCpu->CTX_SUFF(pVM);
10000 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10001 if (RT_LIKELY(rc == VINF_SUCCESS))
10002 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10003 else
10004 {
10005 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10006 rc = VERR_EM_INTERPRETER;
10007 }
10008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10009 return rc;
10010}
10011
10012
10013/**
10014 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10015 */
10016HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10017{
10018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10019 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10020 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10021 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10022 AssertRCReturn(rc, rc);
10023
10024 PVM pVM = pVCpu->CTX_SUFF(pVM);
10025 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10026 rc = VBOXSTRICTRC_VAL(rc2);
10027 if (RT_LIKELY( rc == VINF_SUCCESS
10028 || rc == VINF_EM_HALT))
10029 {
10030 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10031 AssertRCReturn(rc3, rc3);
10032
10033 if ( rc == VINF_EM_HALT
10034 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10035 {
10036 rc = VINF_SUCCESS;
10037 }
10038 }
10039 else
10040 {
10041 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10042 rc = VERR_EM_INTERPRETER;
10043 }
10044 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10045 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10046 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10047 return rc;
10048}
10049
10050
10051/**
10052 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10053 */
10054HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10055{
10056 /*
10057 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10058 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10059 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10060 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10061 */
10062 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10063 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10064 HMVMX_RETURN_UNEXPECTED_EXIT();
10065}
10066
10067
10068/**
10069 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10070 */
10071HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10072{
10073 /*
10074 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10075 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10076 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10077 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10078 */
10079 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10080 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10081 HMVMX_RETURN_UNEXPECTED_EXIT();
10082}
10083
10084
10085/**
10086 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10087 */
10088HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10089{
10090 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10091 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10092 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10093 HMVMX_RETURN_UNEXPECTED_EXIT();
10094}
10095
10096
10097/**
10098 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10099 */
10100HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10101{
10102 /*
10103 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10104 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10105 * See Intel spec. 25.3 "Other Causes of VM-exits".
10106 */
10107 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10108 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10109 HMVMX_RETURN_UNEXPECTED_EXIT();
10110}
10111
10112
10113/**
10114 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10115 * VM-exit.
10116 */
10117HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10118{
10119 /*
10120 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10121 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10122 *
10123 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10124 * See Intel spec. "23.8 Restrictions on VMX operation".
10125 */
10126 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10127 return VINF_SUCCESS;
10128}
10129
10130
10131/**
10132 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10133 * VM-exit.
10134 */
10135HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10136{
10137 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10138 return VINF_EM_RESET;
10139}
10140
10141
10142/**
10143 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10144 */
10145HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10146{
10147 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10148 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10149 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10150 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10151 AssertRCReturn(rc, rc);
10152
10153 pMixedCtx->rip++;
10154 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10155 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10156 rc = VINF_SUCCESS;
10157 else
10158 rc = VINF_EM_HALT;
10159
10160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10161 return rc;
10162}
10163
10164
10165/**
10166 * VM-exit handler for instructions that result in a #UD exception delivered to
10167 * the guest.
10168 */
10169HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10170{
10171 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10172 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10173 return VINF_SUCCESS;
10174}
10175
10176
10177/**
10178 * VM-exit handler for expiry of the VMX preemption timer.
10179 */
10180HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10181{
10182 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10183
10184 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10185 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10186
10187 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10188 PVM pVM = pVCpu->CTX_SUFF(pVM);
10189 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10190 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10191 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10192}
10193
10194
10195/**
10196 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10197 */
10198HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10199{
10200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10201
10202 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10203 /** @todo check if XSETBV is supported by the recompiler. */
10204 return VERR_EM_INTERPRETER;
10205}
10206
10207
10208/**
10209 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10210 */
10211HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10212{
10213 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10214
10215 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10216 /** @todo implement EMInterpretInvpcid() */
10217 return VERR_EM_INTERPRETER;
10218}
10219
10220
10221/**
10222 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10223 * Error VM-exit.
10224 */
10225HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10226{
10227 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10228 AssertRCReturn(rc, rc);
10229
10230 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10231 AssertRCReturn(rc, rc);
10232
10233 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10234 NOREF(uInvalidReason);
10235
10236#ifdef VBOX_STRICT
10237 uint32_t uIntrState;
10238 HMVMXHCUINTREG uHCReg;
10239 uint64_t u64Val;
10240 uint32_t u32Val;
10241
10242 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10243 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10244 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10245 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10246 AssertRCReturn(rc, rc);
10247
10248 Log4(("uInvalidReason %u\n", uInvalidReason));
10249 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10250 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10251 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10252 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10253
10254 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10255 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10256 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10257 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10258 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10259 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10260 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10261 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10262 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10263 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10264 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10265 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10266#else
10267 NOREF(pVmxTransient);
10268#endif
10269
10270 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10271 return VERR_VMX_INVALID_GUEST_STATE;
10272}
10273
10274
10275/**
10276 * VM-exit handler for VM-entry failure due to an MSR-load
10277 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10278 */
10279HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10280{
10281 NOREF(pVmxTransient);
10282 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10283 HMVMX_RETURN_UNEXPECTED_EXIT();
10284}
10285
10286
10287/**
10288 * VM-exit handler for VM-entry failure due to a machine-check event
10289 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10290 */
10291HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10292{
10293 NOREF(pVmxTransient);
10294 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10295 HMVMX_RETURN_UNEXPECTED_EXIT();
10296}
10297
10298
10299/**
10300 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10301 * theory.
10302 */
10303HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10304{
10305 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10306 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10307 return VERR_VMX_UNDEFINED_EXIT_CODE;
10308}
10309
10310
10311/**
10312 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10313 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10314 * Conditional VM-exit.
10315 */
10316HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10317{
10318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10319
10320 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10322 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10323 return VERR_EM_INTERPRETER;
10324 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10325 HMVMX_RETURN_UNEXPECTED_EXIT();
10326}
10327
10328
10329/**
10330 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10331 */
10332HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10333{
10334 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10335
10336 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10337 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10338 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10339 return VERR_EM_INTERPRETER;
10340 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10341 HMVMX_RETURN_UNEXPECTED_EXIT();
10342}
10343
10344
10345/**
10346 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10347 */
10348HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10349{
10350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10351
10352 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10353 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10354 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10355 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10356 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10357 {
10358 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10359 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10360 }
10361 AssertRCReturn(rc, rc);
10362 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10363
10364#ifdef VBOX_STRICT
10365 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10366 {
10367 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10368 && pMixedCtx->ecx != MSR_K6_EFER)
10369 {
10370 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10371 HMVMX_RETURN_UNEXPECTED_EXIT();
10372 }
10373# if HC_ARCH_BITS == 64
10374 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10375 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10376 {
10377 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10378 HMVMX_RETURN_UNEXPECTED_EXIT();
10379 }
10380# endif
10381 }
10382#endif
10383
10384 PVM pVM = pVCpu->CTX_SUFF(pVM);
10385 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10386 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10387 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10389 if (RT_LIKELY(rc == VINF_SUCCESS))
10390 {
10391 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10392 Assert(pVmxTransient->cbInstr == 2);
10393 }
10394 return rc;
10395}
10396
10397
10398/**
10399 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10400 */
10401HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10402{
10403 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10404 PVM pVM = pVCpu->CTX_SUFF(pVM);
10405 int rc = VINF_SUCCESS;
10406
10407 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10408 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10409 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10410 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10411 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10412 {
10413 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10414 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10415 }
10416 AssertRCReturn(rc, rc);
10417 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10418
10419 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10420 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10421 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10422
10423 if (RT_LIKELY(rc == VINF_SUCCESS))
10424 {
10425 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10426
10427 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10428 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10429 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10430 {
10431 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10432 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10433 EMInterpretWrmsr() changes it. */
10434 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10435 }
10436 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10437 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10438 else if (pMixedCtx->ecx == MSR_K6_EFER)
10439 {
10440 /*
10441 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10442 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10443 * the other bits as well, SCE and NXE. See @bugref{7368}.
10444 */
10445 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10446 }
10447
10448 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10449 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10450 {
10451 switch (pMixedCtx->ecx)
10452 {
10453 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10454 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10455 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10456 case MSR_K8_FS_BASE: /* no break */
10457 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10458 case MSR_K6_EFER: /* already handled above */ break;
10459 default:
10460 {
10461 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10462 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10463#if HC_ARCH_BITS == 64
10464 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10465 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10466#endif
10467 break;
10468 }
10469 }
10470 }
10471#ifdef VBOX_STRICT
10472 else
10473 {
10474 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10475 switch (pMixedCtx->ecx)
10476 {
10477 case MSR_IA32_SYSENTER_CS:
10478 case MSR_IA32_SYSENTER_EIP:
10479 case MSR_IA32_SYSENTER_ESP:
10480 case MSR_K8_FS_BASE:
10481 case MSR_K8_GS_BASE:
10482 {
10483 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10484 HMVMX_RETURN_UNEXPECTED_EXIT();
10485 }
10486
10487 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10488 default:
10489 {
10490 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10491 {
10492 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10493 if (pMixedCtx->ecx != MSR_K6_EFER)
10494 {
10495 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10496 pMixedCtx->ecx));
10497 HMVMX_RETURN_UNEXPECTED_EXIT();
10498 }
10499 }
10500
10501#if HC_ARCH_BITS == 64
10502 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10503 {
10504 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10505 HMVMX_RETURN_UNEXPECTED_EXIT();
10506 }
10507#endif
10508 break;
10509 }
10510 }
10511 }
10512#endif /* VBOX_STRICT */
10513 }
10514 return rc;
10515}
10516
10517
10518/**
10519 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10520 */
10521HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10522{
10523 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10524
10525 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10526 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10527 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10528 return VERR_EM_INTERPRETER;
10529 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10530 HMVMX_RETURN_UNEXPECTED_EXIT();
10531}
10532
10533
10534/**
10535 * VM-exit handler for when the TPR value is lowered below the specified
10536 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10537 */
10538HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10539{
10540 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10541 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10542
10543 /*
10544 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10545 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10546 * resume guest execution.
10547 */
10548 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10549 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10550 return VINF_SUCCESS;
10551}
10552
10553
10554/**
10555 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10556 * VM-exit.
10557 *
10558 * @retval VINF_SUCCESS when guest execution can continue.
10559 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10560 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10561 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10562 * recompiler.
10563 */
10564HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10565{
10566 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10567 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10568 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10569 AssertRCReturn(rc, rc);
10570
10571 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10572 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10573 PVM pVM = pVCpu->CTX_SUFF(pVM);
10574 switch (uAccessType)
10575 {
10576 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10577 {
10578#if 0
10579 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10580 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10581#else
10582 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10583 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10584 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10585#endif
10586 AssertRCReturn(rc, rc);
10587
10588 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10589 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10590 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10591 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10592
10593 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10594 {
10595 case 0: /* CR0 */
10596 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10597 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10598 break;
10599 case 2: /* CR2 */
10600 /* Nothing to do here, CR2 it's not part of the VMCS. */
10601 break;
10602 case 3: /* CR3 */
10603 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10604 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10605 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10606 break;
10607 case 4: /* CR4 */
10608 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10609 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10610 break;
10611 case 8: /* CR8 */
10612 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10613 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10614 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10615 break;
10616 default:
10617 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10618 break;
10619 }
10620
10621 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10622 break;
10623 }
10624
10625 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10626 {
10627 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10628 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10629 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10630 AssertRCReturn(rc, rc);
10631 Assert( !pVM->hm.s.fNestedPaging
10632 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10633 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10634
10635 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10636 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10637 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10638
10639 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10640 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10641 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10642 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10643 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10644 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10645 break;
10646 }
10647
10648 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10649 {
10650 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10651 AssertRCReturn(rc, rc);
10652 rc = EMInterpretCLTS(pVM, pVCpu);
10653 AssertRCReturn(rc, rc);
10654 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10656 Log4(("CRX CLTS write rc=%d\n", rc));
10657 break;
10658 }
10659
10660 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10661 {
10662 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10663 AssertRCReturn(rc, rc);
10664 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10665 if (RT_LIKELY(rc == VINF_SUCCESS))
10666 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10667 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10668 Log4(("CRX LMSW write rc=%d\n", rc));
10669 break;
10670 }
10671
10672 default:
10673 {
10674 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10675 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10676 }
10677 }
10678
10679 /* Validate possible error codes. */
10680 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10681 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10682 if (RT_SUCCESS(rc))
10683 {
10684 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10685 AssertRCReturn(rc2, rc2);
10686 }
10687
10688 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10689 return rc;
10690}
10691
10692
10693/**
10694 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10695 * VM-exit.
10696 */
10697HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10698{
10699 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10700 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10701
10702 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10703 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10704 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10705 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10706 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10707 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10708 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10709 AssertRCReturn(rc2, rc2);
10710
10711 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10712 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10713 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10714 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10715 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10716 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10717 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10718 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10719
10720 /* I/O operation lookup arrays. */
10721 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10722 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10723
10724 VBOXSTRICTRC rcStrict;
10725 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10726 const uint32_t cbInstr = pVmxTransient->cbInstr;
10727 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10728 PVM pVM = pVCpu->CTX_SUFF(pVM);
10729 if (fIOString)
10730 {
10731#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10732 /*
10733 * INS/OUTS - I/O String instruction.
10734 *
10735 * Use instruction-information if available, otherwise fall back on
10736 * interpreting the instruction.
10737 */
10738 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10739 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10740 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10741 {
10742 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10743 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10744 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10745 AssertRCReturn(rc2, rc2);
10746 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10747 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10748 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10749 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10750 if (fIOWrite)
10751 {
10752 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10753 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10754 }
10755 else
10756 {
10757 /*
10758 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10759 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10760 * See Intel Instruction spec. for "INS".
10761 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10762 */
10763 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10764 }
10765 }
10766 else
10767 {
10768 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10769 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10770 AssertRCReturn(rc2, rc2);
10771 rcStrict = IEMExecOne(pVCpu);
10772 }
10773 /** @todo IEM needs to be setting these flags somehow. */
10774 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10775 fUpdateRipAlready = true;
10776#else
10777 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10778 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10779 if (RT_SUCCESS(rcStrict))
10780 {
10781 if (fIOWrite)
10782 {
10783 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10784 (DISCPUMODE)pDis->uAddrMode, cbValue);
10785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10786 }
10787 else
10788 {
10789 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10790 (DISCPUMODE)pDis->uAddrMode, cbValue);
10791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10792 }
10793 }
10794 else
10795 {
10796 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10797 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10798 }
10799#endif
10800 }
10801 else
10802 {
10803 /*
10804 * IN/OUT - I/O instruction.
10805 */
10806 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10807 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10808 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10809 if (fIOWrite)
10810 {
10811 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10812 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10813 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10814 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10815 }
10816 else
10817 {
10818 uint32_t u32Result = 0;
10819 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10820 if (IOM_SUCCESS(rcStrict))
10821 {
10822 /* Save result of I/O IN instr. in AL/AX/EAX. */
10823 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10824 }
10825 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10826 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10827 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10828 }
10829 }
10830
10831 if (IOM_SUCCESS(rcStrict))
10832 {
10833 if (!fUpdateRipAlready)
10834 {
10835 pMixedCtx->rip += cbInstr;
10836 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10837 }
10838
10839 /*
10840 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10841 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10842 */
10843 if (fIOString)
10844 {
10845 /** @todo Single-step for INS/OUTS with REP prefix? */
10846 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10847 }
10848 else if (fStepping)
10849 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10850
10851 /*
10852 * If any I/O breakpoints are armed, we need to check if one triggered
10853 * and take appropriate action.
10854 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10855 */
10856 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10857 AssertRCReturn(rc2, rc2);
10858
10859 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10860 * execution engines about whether hyper BPs and such are pending. */
10861 uint32_t const uDr7 = pMixedCtx->dr[7];
10862 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10863 && X86_DR7_ANY_RW_IO(uDr7)
10864 && (pMixedCtx->cr4 & X86_CR4_DE))
10865 || DBGFBpIsHwIoArmed(pVM)))
10866 {
10867 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10868
10869 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10870 VMMRZCallRing3Disable(pVCpu);
10871 HM_DISABLE_PREEMPT_IF_NEEDED();
10872
10873 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10874
10875 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10876 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10877 {
10878 /* Raise #DB. */
10879 if (fIsGuestDbgActive)
10880 ASMSetDR6(pMixedCtx->dr[6]);
10881 if (pMixedCtx->dr[7] != uDr7)
10882 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10883
10884 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10885 }
10886 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10887 else if ( rcStrict2 != VINF_SUCCESS
10888 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10889 rcStrict = rcStrict2;
10890
10891 HM_RESTORE_PREEMPT_IF_NEEDED();
10892 VMMRZCallRing3Enable(pVCpu);
10893 }
10894 }
10895
10896#ifdef DEBUG
10897 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10898 Assert(!fIOWrite);
10899 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10900 Assert(fIOWrite);
10901 else
10902 {
10903 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10904 * statuses, that the VMM device and some others may return. See
10905 * IOM_SUCCESS() for guidance. */
10906 AssertMsg( RT_FAILURE(rcStrict)
10907 || rcStrict == VINF_SUCCESS
10908 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10909 || rcStrict == VINF_EM_DBG_BREAKPOINT
10910 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10911 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10912 }
10913#endif
10914
10915 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10916 return VBOXSTRICTRC_TODO(rcStrict);
10917}
10918
10919
10920/**
10921 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10922 * VM-exit.
10923 */
10924HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10925{
10926 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10927
10928 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10929 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10930 AssertRCReturn(rc, rc);
10931 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10932 {
10933 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10934 AssertRCReturn(rc, rc);
10935 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10936 {
10937 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10938
10939 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10940 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10941
10942 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10943 Assert(!pVCpu->hm.s.Event.fPending);
10944 pVCpu->hm.s.Event.fPending = true;
10945 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10946 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10947 AssertRCReturn(rc, rc);
10948 if (fErrorCodeValid)
10949 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10950 else
10951 pVCpu->hm.s.Event.u32ErrCode = 0;
10952 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10953 && uVector == X86_XCPT_PF)
10954 {
10955 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10956 }
10957
10958 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10960 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10961 }
10962 }
10963
10964 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10965 * emulation. */
10966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10967 return VERR_EM_INTERPRETER;
10968}
10969
10970
10971/**
10972 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10973 */
10974HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10975{
10976 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10977 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10978 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10979 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10980 AssertRCReturn(rc, rc);
10981 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10982 return VINF_EM_DBG_STEPPED;
10983}
10984
10985
10986/**
10987 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10988 */
10989HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10990{
10991 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10992
10993 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10994 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10995 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10996 return VINF_SUCCESS;
10997 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10998 return rc;
10999
11000#if 0
11001 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11002 * just sync the whole thing. */
11003 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11004#else
11005 /* Aggressive state sync. for now. */
11006 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11007 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11008 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11009#endif
11010 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11011 AssertRCReturn(rc, rc);
11012
11013 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11014 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11015 switch (uAccessType)
11016 {
11017 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11018 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11019 {
11020 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11021 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11022 {
11023 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11024 }
11025
11026 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11027 GCPhys &= PAGE_BASE_GC_MASK;
11028 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11029 PVM pVM = pVCpu->CTX_SUFF(pVM);
11030 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11031 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11032
11033 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11034 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11035 CPUMCTX2CORE(pMixedCtx), GCPhys);
11036 rc = VBOXSTRICTRC_VAL(rc2);
11037 Log4(("ApicAccess rc=%d\n", rc));
11038 if ( rc == VINF_SUCCESS
11039 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11040 || rc == VERR_PAGE_NOT_PRESENT)
11041 {
11042 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11043 | HM_CHANGED_GUEST_RSP
11044 | HM_CHANGED_GUEST_RFLAGS
11045 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11046 rc = VINF_SUCCESS;
11047 }
11048 break;
11049 }
11050
11051 default:
11052 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11053 rc = VINF_EM_RAW_EMULATE_INSTR;
11054 break;
11055 }
11056
11057 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11058 return rc;
11059}
11060
11061
11062/**
11063 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11064 * VM-exit.
11065 */
11066HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11067{
11068 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11069
11070 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11071 if (pVmxTransient->fWasGuestDebugStateActive)
11072 {
11073 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11074 HMVMX_RETURN_UNEXPECTED_EXIT();
11075 }
11076
11077 int rc = VERR_INTERNAL_ERROR_5;
11078 if ( !DBGFIsStepping(pVCpu)
11079 && !pVCpu->hm.s.fSingleInstruction
11080 && !pVmxTransient->fWasHyperDebugStateActive)
11081 {
11082 /* Don't intercept MOV DRx and #DB any more. */
11083 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11084 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11085 AssertRCReturn(rc, rc);
11086
11087 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11088 {
11089#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11090 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11091 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11092 AssertRCReturn(rc, rc);
11093#endif
11094 }
11095
11096 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11097 VMMRZCallRing3Disable(pVCpu);
11098 HM_DISABLE_PREEMPT_IF_NEEDED();
11099
11100 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11101 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11102 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11103
11104 HM_RESTORE_PREEMPT_IF_NEEDED();
11105 VMMRZCallRing3Enable(pVCpu);
11106
11107#ifdef VBOX_WITH_STATISTICS
11108 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11109 AssertRCReturn(rc, rc);
11110 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11112 else
11113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11114#endif
11115 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11116 return VINF_SUCCESS;
11117 }
11118
11119 /*
11120 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11121 * Update the segment registers and DR7 from the CPU.
11122 */
11123 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11124 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11125 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11126 AssertRCReturn(rc, rc);
11127 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11128
11129 PVM pVM = pVCpu->CTX_SUFF(pVM);
11130 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11131 {
11132 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11133 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11134 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11135 if (RT_SUCCESS(rc))
11136 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11137 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11138 }
11139 else
11140 {
11141 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11142 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11143 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11144 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11145 }
11146
11147 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11148 if (RT_SUCCESS(rc))
11149 {
11150 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11151 AssertRCReturn(rc2, rc2);
11152 }
11153 return rc;
11154}
11155
11156
11157/**
11158 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11159 * Conditional VM-exit.
11160 */
11161HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11162{
11163 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11164 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11165
11166 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11167 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11168 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11169 return VINF_SUCCESS;
11170 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11171 return rc;
11172
11173 RTGCPHYS GCPhys = 0;
11174 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11175
11176#if 0
11177 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11178#else
11179 /* Aggressive state sync. for now. */
11180 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11181 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11182 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11183#endif
11184 AssertRCReturn(rc, rc);
11185
11186 /*
11187 * If we succeed, resume guest execution.
11188 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11189 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11190 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11191 * weird case. See @bugref{6043}.
11192 */
11193 PVM pVM = pVCpu->CTX_SUFF(pVM);
11194 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11195 rc = VBOXSTRICTRC_VAL(rc2);
11196 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11197 if ( rc == VINF_SUCCESS
11198 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11199 || rc == VERR_PAGE_NOT_PRESENT)
11200 {
11201 /* Successfully handled MMIO operation. */
11202 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11203 | HM_CHANGED_GUEST_RSP
11204 | HM_CHANGED_GUEST_RFLAGS
11205 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11206 rc = VINF_SUCCESS;
11207 }
11208 return rc;
11209}
11210
11211
11212/**
11213 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11214 * VM-exit.
11215 */
11216HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11217{
11218 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11219 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11220
11221 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11222 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11223 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11224 return VINF_SUCCESS;
11225 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11226 return rc;
11227
11228 RTGCPHYS GCPhys = 0;
11229 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11230 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11231#if 0
11232 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11233#else
11234 /* Aggressive state sync. for now. */
11235 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11236 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11237 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11238#endif
11239 AssertRCReturn(rc, rc);
11240
11241 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11242 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11243
11244 RTGCUINT uErrorCode = 0;
11245 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11246 uErrorCode |= X86_TRAP_PF_ID;
11247 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11248 uErrorCode |= X86_TRAP_PF_RW;
11249 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11250 uErrorCode |= X86_TRAP_PF_P;
11251
11252 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11253
11254 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11255 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11256
11257 /* Handle the pagefault trap for the nested shadow table. */
11258 PVM pVM = pVCpu->CTX_SUFF(pVM);
11259 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11260 TRPMResetTrap(pVCpu);
11261
11262 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11263 if ( rc == VINF_SUCCESS
11264 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11265 || rc == VERR_PAGE_NOT_PRESENT)
11266 {
11267 /* Successfully synced our nested page tables. */
11268 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11269 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11270 | HM_CHANGED_GUEST_RSP
11271 | HM_CHANGED_GUEST_RFLAGS);
11272 return VINF_SUCCESS;
11273 }
11274
11275 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11276 return rc;
11277}
11278
11279/** @} */
11280
11281/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11282/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11283/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11284
11285/** @name VM-exit exception handlers.
11286 * @{
11287 */
11288
11289/**
11290 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11291 */
11292static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11293{
11294 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11296
11297 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11298 AssertRCReturn(rc, rc);
11299
11300 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11301 {
11302 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11303 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11304
11305 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11306 * provides VM-exit instruction length. If this causes problem later,
11307 * disassemble the instruction like it's done on AMD-V. */
11308 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11309 AssertRCReturn(rc2, rc2);
11310 return rc;
11311 }
11312
11313 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11314 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11315 return rc;
11316}
11317
11318
11319/**
11320 * VM-exit exception handler for #BP (Breakpoint exception).
11321 */
11322static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11323{
11324 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11325 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11326
11327 /** @todo Try optimize this by not saving the entire guest state unless
11328 * really needed. */
11329 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11330 AssertRCReturn(rc, rc);
11331
11332 PVM pVM = pVCpu->CTX_SUFF(pVM);
11333 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11334 if (rc == VINF_EM_RAW_GUEST_TRAP)
11335 {
11336 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11337 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11338 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11339 AssertRCReturn(rc, rc);
11340
11341 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11342 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11343 }
11344
11345 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11346 return rc;
11347}
11348
11349
11350/**
11351 * VM-exit exception handler for #DB (Debug exception).
11352 */
11353static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11354{
11355 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11357 Log6(("XcptDB\n"));
11358
11359 /*
11360 * Get the DR6-like values from the exit qualification and pass it to DBGF
11361 * for processing.
11362 */
11363 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11364 AssertRCReturn(rc, rc);
11365
11366 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11367 uint64_t uDR6 = X86_DR6_INIT_VAL;
11368 uDR6 |= ( pVmxTransient->uExitQualification
11369 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11370
11371 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11372 if (rc == VINF_EM_RAW_GUEST_TRAP)
11373 {
11374 /*
11375 * The exception was for the guest. Update DR6, DR7.GD and
11376 * IA32_DEBUGCTL.LBR before forwarding it.
11377 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11378 */
11379 VMMRZCallRing3Disable(pVCpu);
11380 HM_DISABLE_PREEMPT_IF_NEEDED();
11381
11382 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11383 pMixedCtx->dr[6] |= uDR6;
11384 if (CPUMIsGuestDebugStateActive(pVCpu))
11385 ASMSetDR6(pMixedCtx->dr[6]);
11386
11387 HM_RESTORE_PREEMPT_IF_NEEDED();
11388 VMMRZCallRing3Enable(pVCpu);
11389
11390 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11391 AssertRCReturn(rc, rc);
11392
11393 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11394 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11395
11396 /* Paranoia. */
11397 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11398 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11399
11400 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11401 AssertRCReturn(rc, rc);
11402
11403 /*
11404 * Raise #DB in the guest.
11405 *
11406 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11407 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11408 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11409 *
11410 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11411 */
11412 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11413 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11414 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11415 AssertRCReturn(rc, rc);
11416 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11417 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11418 return VINF_SUCCESS;
11419 }
11420
11421 /*
11422 * Not a guest trap, must be a hypervisor related debug event then.
11423 * Update DR6 in case someone is interested in it.
11424 */
11425 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11426 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11427 CPUMSetHyperDR6(pVCpu, uDR6);
11428
11429 return rc;
11430}
11431
11432
11433/**
11434 * VM-exit exception handler for #NM (Device-not-available exception: floating
11435 * point exception).
11436 */
11437static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11438{
11439 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11440
11441 /* We require CR0 and EFER. EFER is always up-to-date. */
11442 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11443 AssertRCReturn(rc, rc);
11444
11445 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11446 VMMRZCallRing3Disable(pVCpu);
11447 HM_DISABLE_PREEMPT_IF_NEEDED();
11448
11449 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11450 if (pVmxTransient->fWasGuestFPUStateActive)
11451 {
11452 rc = VINF_EM_RAW_GUEST_TRAP;
11453 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11454 }
11455 else
11456 {
11457#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11458 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11459#endif
11460 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11461 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11462 }
11463
11464 HM_RESTORE_PREEMPT_IF_NEEDED();
11465 VMMRZCallRing3Enable(pVCpu);
11466
11467 if (rc == VINF_SUCCESS)
11468 {
11469 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11470 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11471 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11472 pVCpu->hm.s.fUseGuestFpu = true;
11473 }
11474 else
11475 {
11476 /* Forward #NM to the guest. */
11477 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11478 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11479 AssertRCReturn(rc, rc);
11480 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11481 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11483 }
11484
11485 return VINF_SUCCESS;
11486}
11487
11488
11489/**
11490 * VM-exit exception handler for #GP (General-protection exception).
11491 *
11492 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11493 */
11494static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11495{
11496 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11498
11499 int rc = VERR_INTERNAL_ERROR_5;
11500 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11501 {
11502#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11503 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11504 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11505 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11506 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11507 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11508 AssertRCReturn(rc, rc);
11509 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11510 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11511 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11512 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11513 return rc;
11514#else
11515 /* We don't intercept #GP. */
11516 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11517 NOREF(pVmxTransient);
11518 return VERR_VMX_UNEXPECTED_EXCEPTION;
11519#endif
11520 }
11521
11522 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11523 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11524
11525 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11526 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11527 AssertRCReturn(rc, rc);
11528
11529 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11530 uint32_t cbOp = 0;
11531 PVM pVM = pVCpu->CTX_SUFF(pVM);
11532 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11533 if (RT_SUCCESS(rc))
11534 {
11535 rc = VINF_SUCCESS;
11536 Assert(cbOp == pDis->cbInstr);
11537 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11538 switch (pDis->pCurInstr->uOpcode)
11539 {
11540 case OP_CLI:
11541 {
11542 pMixedCtx->eflags.Bits.u1IF = 0;
11543 pMixedCtx->rip += pDis->cbInstr;
11544 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11545 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11547 break;
11548 }
11549
11550 case OP_STI:
11551 {
11552 pMixedCtx->eflags.Bits.u1IF = 1;
11553 pMixedCtx->rip += pDis->cbInstr;
11554 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11555 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11556 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11557 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11558 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11559 break;
11560 }
11561
11562 case OP_HLT:
11563 {
11564 rc = VINF_EM_HALT;
11565 pMixedCtx->rip += pDis->cbInstr;
11566 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11568 break;
11569 }
11570
11571 case OP_POPF:
11572 {
11573 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11574 uint32_t cbParm;
11575 uint32_t uMask;
11576 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11577 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11578 {
11579 cbParm = 4;
11580 uMask = 0xffffffff;
11581 }
11582 else
11583 {
11584 cbParm = 2;
11585 uMask = 0xffff;
11586 }
11587
11588 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11589 RTGCPTR GCPtrStack = 0;
11590 X86EFLAGS Eflags;
11591 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11592 &GCPtrStack);
11593 if (RT_SUCCESS(rc))
11594 {
11595 Assert(sizeof(Eflags.u32) >= cbParm);
11596 Eflags.u32 = 0;
11597 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11598 }
11599 if (RT_FAILURE(rc))
11600 {
11601 rc = VERR_EM_INTERPRETER;
11602 break;
11603 }
11604 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11605 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11606 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11607 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11608 pMixedCtx->esp += cbParm;
11609 pMixedCtx->esp &= uMask;
11610 pMixedCtx->rip += pDis->cbInstr;
11611 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11612 | HM_CHANGED_GUEST_RSP
11613 | HM_CHANGED_GUEST_RFLAGS);
11614 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11615 if (fStepping)
11616 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11617
11618 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11619 break;
11620 }
11621
11622 case OP_PUSHF:
11623 {
11624 uint32_t cbParm;
11625 uint32_t uMask;
11626 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11627 {
11628 cbParm = 4;
11629 uMask = 0xffffffff;
11630 }
11631 else
11632 {
11633 cbParm = 2;
11634 uMask = 0xffff;
11635 }
11636
11637 /* Get the stack pointer & push the contents of eflags onto the stack. */
11638 RTGCPTR GCPtrStack = 0;
11639 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11640 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11641 if (RT_FAILURE(rc))
11642 {
11643 rc = VERR_EM_INTERPRETER;
11644 break;
11645 }
11646 X86EFLAGS Eflags = pMixedCtx->eflags;
11647 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11648 Eflags.Bits.u1RF = 0;
11649 Eflags.Bits.u1VM = 0;
11650
11651 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11652 if (RT_FAILURE(rc))
11653 {
11654 rc = VERR_EM_INTERPRETER;
11655 break;
11656 }
11657 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11658 pMixedCtx->esp -= cbParm;
11659 pMixedCtx->esp &= uMask;
11660 pMixedCtx->rip += pDis->cbInstr;
11661 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11662 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11663 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11664 break;
11665 }
11666
11667 case OP_IRET:
11668 {
11669 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11670 * instruction reference. */
11671 RTGCPTR GCPtrStack = 0;
11672 uint32_t uMask = 0xffff;
11673 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11674 uint16_t aIretFrame[3];
11675 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11676 {
11677 rc = VERR_EM_INTERPRETER;
11678 break;
11679 }
11680 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11681 &GCPtrStack);
11682 if (RT_SUCCESS(rc))
11683 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11684 if (RT_FAILURE(rc))
11685 {
11686 rc = VERR_EM_INTERPRETER;
11687 break;
11688 }
11689 pMixedCtx->eip = 0;
11690 pMixedCtx->ip = aIretFrame[0];
11691 pMixedCtx->cs.Sel = aIretFrame[1];
11692 pMixedCtx->cs.ValidSel = aIretFrame[1];
11693 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11694 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11695 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11696 pMixedCtx->sp += sizeof(aIretFrame);
11697 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11698 | HM_CHANGED_GUEST_SEGMENT_REGS
11699 | HM_CHANGED_GUEST_RSP
11700 | HM_CHANGED_GUEST_RFLAGS);
11701 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11702 if (fStepping)
11703 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11704 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11706 break;
11707 }
11708
11709 case OP_INT:
11710 {
11711 uint16_t uVector = pDis->Param1.uValue & 0xff;
11712 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11713 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11715 break;
11716 }
11717
11718 case OP_INTO:
11719 {
11720 if (pMixedCtx->eflags.Bits.u1OF)
11721 {
11722 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11723 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11725 }
11726 break;
11727 }
11728
11729 default:
11730 {
11731 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11732 EMCODETYPE_SUPERVISOR);
11733 rc = VBOXSTRICTRC_VAL(rc2);
11734 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11735 /** @todo We have to set pending-debug exceptions here when the guest is
11736 * single-stepping depending on the instruction that was interpreted. */
11737 Log4(("#GP rc=%Rrc\n", rc));
11738 break;
11739 }
11740 }
11741 }
11742 else
11743 rc = VERR_EM_INTERPRETER;
11744
11745 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11746 ("#GP Unexpected rc=%Rrc\n", rc));
11747 return rc;
11748}
11749
11750
11751#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11752/**
11753 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11754 * the exception reported in the VMX transient structure back into the VM.
11755 *
11756 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11757 * up-to-date.
11758 */
11759static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11760{
11761 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11762
11763 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11764 hmR0VmxCheckExitDueToEventDelivery(). */
11765 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11766 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11767 AssertRCReturn(rc, rc);
11768 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11769
11770#ifdef DEBUG_ramshankar
11771 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11772 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11773 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11774#endif
11775
11776 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11777 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11778 return VINF_SUCCESS;
11779}
11780#endif
11781
11782
11783/**
11784 * VM-exit exception handler for #PF (Page-fault exception).
11785 */
11786static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11787{
11788 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11789 PVM pVM = pVCpu->CTX_SUFF(pVM);
11790 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11791 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11792 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11793 AssertRCReturn(rc, rc);
11794
11795#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11796 if (pVM->hm.s.fNestedPaging)
11797 {
11798 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11799 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11800 {
11801 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11802 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11803 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11804 }
11805 else
11806 {
11807 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11808 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11809 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11810 }
11811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11812 return rc;
11813 }
11814#else
11815 Assert(!pVM->hm.s.fNestedPaging);
11816 NOREF(pVM);
11817#endif
11818
11819 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11820 AssertRCReturn(rc, rc);
11821
11822 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11823 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11824
11825 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11826 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11827 (RTGCPTR)pVmxTransient->uExitQualification);
11828
11829 Log4(("#PF: rc=%Rrc\n", rc));
11830 if (rc == VINF_SUCCESS)
11831 {
11832 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11833 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11834 * memory? We don't update the whole state here... */
11835 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11836 | HM_CHANGED_GUEST_RSP
11837 | HM_CHANGED_GUEST_RFLAGS
11838 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11839 TRPMResetTrap(pVCpu);
11840 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11841 return rc;
11842 }
11843 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11844 {
11845 if (!pVmxTransient->fVectoringPF)
11846 {
11847 /* It's a guest page fault and needs to be reflected to the guest. */
11848 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11849 TRPMResetTrap(pVCpu);
11850 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11851 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11852 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11853 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11854 }
11855 else
11856 {
11857 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11858 TRPMResetTrap(pVCpu);
11859 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11860 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11861 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11862 }
11863
11864 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11865 return VINF_SUCCESS;
11866 }
11867
11868 TRPMResetTrap(pVCpu);
11869 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11870 return rc;
11871}
11872
11873/** @} */
11874
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette