VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51853

Last change on this file since 51853 was 51757, checked in by vboxsync, 11 years ago

VMM: Change error codes to be consistent.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 491.6 KB
Line 
1/* $Id: HMVMXR0.cpp 51757 2014-06-30 06:19:51Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#include <VBox/vmm/gim.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#ifdef DEBUG_ramshankar
41# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
42# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_CHECK_GUEST_STATE
45# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
46# define HMVMX_ALWAYS_TRAP_PF
47# define HMVMX_ALWAYS_SWAP_FPU_STATE
48# define HMVMX_ALWAYS_FLUSH_TLB
49# define HMVMX_ALWAYS_SWAP_EFER
50#endif
51
52
53/*******************************************************************************
54* Defined Constants And Macros *
55*******************************************************************************/
56#if defined(RT_ARCH_AMD64)
57# define HMVMX_IS_64BIT_HOST_MODE() (true)
58typedef RTHCUINTREG HMVMXHCUINTREG;
59#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
60extern "C" uint32_t g_fVMXIs64bitHost;
61# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
62typedef uint64_t HMVMXHCUINTREG;
63#else
64# define HMVMX_IS_64BIT_HOST_MODE() (false)
65typedef RTHCUINTREG HMVMXHCUINTREG;
66#endif
67
68/** Use the function table. */
69#define HMVMX_USE_FUNCTION_TABLE
70
71/** Determine which tagged-TLB flush handler to use. */
72#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
73#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
74#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
75#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
76
77/** @name Updated-guest-state flags.
78 * @{ */
79#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
80#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
81#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
82#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
83#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
84#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
85#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
86#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
87#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
88#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
89#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
90#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
91#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
92#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
93#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
94#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
95#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
96#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_APIC_STATE)
117/** @} */
118
119/** @name
120 * Flags to skip redundant reads of some common VMCS fields that are not part of
121 * the guest-CPU state but are in the transient structure.
122 */
123#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
130/** @} */
131
132/** @name
133 * States of the VMCS.
134 *
135 * This does not reflect all possible VMCS states but currently only those
136 * needed for maintaining the VMCS consistently even when thread-context hooks
137 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
138 */
139#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
140#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
141#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
142/** @} */
143
144/**
145 * Exception bitmap mask for real-mode guests (real-on-v86).
146 *
147 * We need to intercept all exceptions manually (except #PF). #NM is also
148 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
149 * even in real-mode if we have Nested Paging support.
150 */
151#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
152 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
153 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
154 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
155 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
156 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
157 | RT_BIT(X86_XCPT_XF))
158
159/**
160 * Exception bitmap mask for all contributory exceptions.
161 *
162 * Page fault is deliberately excluded here as it's conditional as to whether
163 * it's contributory or benign. Page faults are handled separately.
164 */
165#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
166 | RT_BIT(X86_XCPT_DE))
167
168/** Maximum VM-instruction error number. */
169#define HMVMX_INSTR_ERROR_MAX 28
170
171/** Profiling macro. */
172#ifdef HM_PROFILE_EXIT_DISPATCH
173# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
175#else
176# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
177# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
178#endif
179
180/** Assert that preemption is disabled or covered by thread-context hooks. */
181#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
183
184/** Assert that we haven't migrated CPUs when thread-context hooks are not
185 * used. */
186#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
188 ("Illegal migration! Entered on CPU %u Current %u\n", \
189 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
190
191/** Helper macro for VM-exit handlers called unexpectedly. */
192#define HMVMX_RETURN_UNEXPECTED_EXIT() \
193 do { \
194 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
195 return VERR_VMX_UNEXPECTED_EXIT; \
196 } while (0)
197
198
199/*******************************************************************************
200* Structures and Typedefs *
201*******************************************************************************/
202/**
203 * VMX transient state.
204 *
205 * A state structure for holding miscellaneous information across
206 * VMX non-root operation and restored after the transition.
207 */
208typedef struct VMXTRANSIENT
209{
210 /** The host's rflags/eflags. */
211 RTCCUINTREG uEflags;
212#if HC_ARCH_BITS == 32
213 uint32_t u32Alignment0;
214#endif
215 /** The guest's TPR value used for TPR shadowing. */
216 uint8_t u8GuestTpr;
217 /** Alignment. */
218 uint8_t abAlignment0[7];
219
220 /** The basic VM-exit reason. */
221 uint16_t uExitReason;
222 /** Alignment. */
223 uint16_t u16Alignment0;
224 /** The VM-exit interruption error code. */
225 uint32_t uExitIntErrorCode;
226 /** The VM-exit exit qualification. */
227 uint64_t uExitQualification;
228
229 /** The VM-exit interruption-information field. */
230 uint32_t uExitIntInfo;
231 /** The VM-exit instruction-length field. */
232 uint32_t cbInstr;
233 /** The VM-exit instruction-information field. */
234 union
235 {
236 /** Plain unsigned int representation. */
237 uint32_t u;
238 /** INS and OUTS information. */
239 struct
240 {
241 uint32_t u6Reserved0 : 7;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u5Reserved1 : 5;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 uint32_t uReserved2 : 14;
248 } StrIo;
249 } ExitInstrInfo;
250 /** Whether the VM-entry failed or not. */
251 bool fVMEntryFailed;
252 /** Alignment. */
253 uint8_t abAlignment1[3];
254
255 /** The VM-entry interruption-information field. */
256 uint32_t uEntryIntInfo;
257 /** The VM-entry exception error code field. */
258 uint32_t uEntryXcptErrorCode;
259 /** The VM-entry instruction length field. */
260 uint32_t cbEntryInstr;
261
262 /** IDT-vectoring information field. */
263 uint32_t uIdtVectoringInfo;
264 /** IDT-vectoring error code. */
265 uint32_t uIdtVectoringErrorCode;
266
267 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
268 uint32_t fVmcsFieldsRead;
269
270 /** Whether the guest FPU was active at the time of VM-exit. */
271 bool fWasGuestFPUStateActive;
272 /** Whether the guest debug state was active at the time of VM-exit. */
273 bool fWasGuestDebugStateActive;
274 /** Whether the hyper debug state was active at the time of VM-exit. */
275 bool fWasHyperDebugStateActive;
276 /** Whether TSC-offsetting should be setup before VM-entry. */
277 bool fUpdateTscOffsettingAndPreemptTimer;
278 /** Whether the VM-exit was caused by a page-fault during delivery of a
279 * contributory exception or a page-fault. */
280 bool fVectoringPF;
281} VMXTRANSIENT;
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287/** Pointer to VMX transient state. */
288typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291/**
292 * MSR-bitmap read permissions.
293 */
294typedef enum VMXMSREXITREAD
295{
296 /** Reading this MSR causes a VM-exit. */
297 VMXMSREXIT_INTERCEPT_READ = 0xb,
298 /** Reading this MSR does not cause a VM-exit. */
299 VMXMSREXIT_PASSTHRU_READ
300} VMXMSREXITREAD;
301/** Pointer to MSR-bitmap read permissions. */
302typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304/**
305 * MSR-bitmap write permissions.
306 */
307typedef enum VMXMSREXITWRITE
308{
309 /** Writing to this MSR causes a VM-exit. */
310 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311 /** Writing to this MSR does not cause a VM-exit. */
312 VMXMSREXIT_PASSTHRU_WRITE
313} VMXMSREXITWRITE;
314/** Pointer to MSR-bitmap write permissions. */
315typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318/**
319 * VMX VM-exit handler.
320 *
321 * @returns VBox status code.
322 * @param pVCpu Pointer to the VMCPU.
323 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
324 * out-of-sync. Make sure to update the required
325 * fields before using them.
326 * @param pVmxTransient Pointer to the VMX-transient structure.
327 */
328#ifndef HMVMX_USE_FUNCTION_TABLE
329typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330#else
331typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332/** Pointer to VM-exit handler. */
333typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334#endif
335
336
337/*******************************************************************************
338* Internal Functions *
339*******************************************************************************/
340static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
341static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
342static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
343 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
344#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
345static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
346#endif
347#ifndef HMVMX_USE_FUNCTION_TABLE
348DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
349# define HMVMX_EXIT_DECL static int
350#else
351# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
352#endif
353
354/** @name VM-exit handlers.
355 * @{
356 */
357static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
358static FNVMXEXITHANDLER hmR0VmxExitExtInt;
359static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
360static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
361static FNVMXEXITHANDLER hmR0VmxExitSipi;
362static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
363static FNVMXEXITHANDLER hmR0VmxExitSmi;
364static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
365static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
366static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369static FNVMXEXITHANDLER hmR0VmxExitHlt;
370static FNVMXEXITHANDLER hmR0VmxExitInvd;
371static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
374static FNVMXEXITHANDLER hmR0VmxExitRsm;
375static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
376static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
377static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
378static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
379static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
380static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
381static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
382static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
383static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
384static FNVMXEXITHANDLER hmR0VmxExitMwait;
385static FNVMXEXITHANDLER hmR0VmxExitMtf;
386static FNVMXEXITHANDLER hmR0VmxExitMonitor;
387static FNVMXEXITHANDLER hmR0VmxExitPause;
388static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
389static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
390static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
391static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
392static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
393static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
394static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
395static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
396static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
397static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
398static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
399static FNVMXEXITHANDLER hmR0VmxExitRdrand;
400static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
401/** @} */
402
403static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
410static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
411#endif
412static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
413
414/*******************************************************************************
415* Global Variables *
416*******************************************************************************/
417#ifdef HMVMX_USE_FUNCTION_TABLE
418
419/**
420 * VMX_EXIT dispatch table.
421 */
422static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
423{
424 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
425 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
426 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
427 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
428 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
429 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
430 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
431 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
432 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
433 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
434 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
435 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
436 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
437 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
438 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
439 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
440 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
441 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
442 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
443 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
444 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
445 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
446 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
447 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
448 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
449 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
450 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
451 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
452 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
453 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
454 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
455 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
456 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
457 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
458 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
459 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
460 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
461 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
462 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
463 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
464 /* 40 UNDEFINED */ hmR0VmxExitPause,
465 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
466 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
467 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
468 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
469 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
470 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
471 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
472 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
473 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
474 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
475 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
476 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
477 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
478 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
479 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
480 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
481 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
482 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
483 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
484};
485#endif /* HMVMX_USE_FUNCTION_TABLE */
486
487#ifdef VBOX_STRICT
488static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
489{
490 /* 0 */ "(Not Used)",
491 /* 1 */ "VMCALL executed in VMX root operation.",
492 /* 2 */ "VMCLEAR with invalid physical address.",
493 /* 3 */ "VMCLEAR with VMXON pointer.",
494 /* 4 */ "VMLAUNCH with non-clear VMCS.",
495 /* 5 */ "VMRESUME with non-launched VMCS.",
496 /* 6 */ "VMRESUME after VMXOFF",
497 /* 7 */ "VM entry with invalid control fields.",
498 /* 8 */ "VM entry with invalid host state fields.",
499 /* 9 */ "VMPTRLD with invalid physical address.",
500 /* 10 */ "VMPTRLD with VMXON pointer.",
501 /* 11 */ "VMPTRLD with incorrect revision identifier.",
502 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
503 /* 13 */ "VMWRITE to read-only VMCS component.",
504 /* 14 */ "(Not Used)",
505 /* 15 */ "VMXON executed in VMX root operation.",
506 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
507 /* 17 */ "VM entry with non-launched executing VMCS.",
508 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
509 /* 19 */ "VMCALL with non-clear VMCS.",
510 /* 20 */ "VMCALL with invalid VM-exit control fields.",
511 /* 21 */ "(Not Used)",
512 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
513 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
514 /* 24 */ "VMCALL with invalid SMM-monitor features.",
515 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
516 /* 26 */ "VM entry with events blocked by MOV SS.",
517 /* 27 */ "(Not Used)",
518 /* 28 */ "Invalid operand to INVEPT/INVVPID."
519};
520#endif /* VBOX_STRICT */
521
522
523
524/**
525 * Updates the VM's last error record. If there was a VMX instruction error,
526 * reads the error data from the VMCS and updates VCPU's last error record as
527 * well.
528 *
529 * @param pVM Pointer to the VM.
530 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
531 * VERR_VMX_UNABLE_TO_START_VM or
532 * VERR_VMX_INVALID_VMCS_FIELD).
533 * @param rc The error code.
534 */
535static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
536{
537 AssertPtr(pVM);
538 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
539 || rc == VERR_VMX_UNABLE_TO_START_VM)
540 {
541 AssertPtrReturnVoid(pVCpu);
542 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
543 }
544 pVM->hm.s.lLastError = rc;
545}
546
547
548/**
549 * Reads the VM-entry interruption-information field from the VMCS into the VMX
550 * transient structure.
551 *
552 * @returns VBox status code.
553 * @param pVmxTransient Pointer to the VMX transient structure.
554 *
555 * @remarks No-long-jump zone!!!
556 */
557DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
558{
559 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
560 AssertRCReturn(rc, rc);
561 return VINF_SUCCESS;
562}
563
564
565/**
566 * Reads the VM-entry exception error code field from the VMCS into
567 * the VMX transient structure.
568 *
569 * @returns VBox status code.
570 * @param pVmxTransient Pointer to the VMX transient structure.
571 *
572 * @remarks No-long-jump zone!!!
573 */
574DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
575{
576 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
577 AssertRCReturn(rc, rc);
578 return VINF_SUCCESS;
579}
580
581
582/**
583 * Reads the VM-entry exception error code field from the VMCS into
584 * the VMX transient structure.
585 *
586 * @returns VBox status code.
587 * @param pVmxTransient Pointer to the VMX transient structure.
588 *
589 * @remarks No-long-jump zone!!!
590 */
591DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
592{
593 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
594 AssertRCReturn(rc, rc);
595 return VINF_SUCCESS;
596}
597
598
599/**
600 * Reads the VM-exit interruption-information field from the VMCS into the VMX
601 * transient structure.
602 *
603 * @returns VBox status code.
604 * @param pVmxTransient Pointer to the VMX transient structure.
605 */
606DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
607{
608 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
609 {
610 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
611 AssertRCReturn(rc, rc);
612 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
613 }
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-exit interruption error code from the VMCS into the VMX
620 * transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 */
625DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
626{
627 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
628 {
629 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
630 AssertRCReturn(rc, rc);
631 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
632 }
633 return VINF_SUCCESS;
634}
635
636
637/**
638 * Reads the VM-exit instruction length field from the VMCS into the VMX
639 * transient structure.
640 *
641 * @returns VBox status code.
642 * @param pVCpu Pointer to the VMCPU.
643 * @param pVmxTransient Pointer to the VMX transient structure.
644 */
645DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
646{
647 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
648 {
649 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
650 AssertRCReturn(rc, rc);
651 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
652 }
653 return VINF_SUCCESS;
654}
655
656
657/**
658 * Reads the VM-exit instruction-information field from the VMCS into
659 * the VMX transient structure.
660 *
661 * @returns VBox status code.
662 * @param pVmxTransient Pointer to the VMX transient structure.
663 */
664DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
665{
666 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
667 {
668 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
669 AssertRCReturn(rc, rc);
670 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
671 }
672 return VINF_SUCCESS;
673}
674
675
676/**
677 * Reads the exit qualification from the VMCS into the VMX transient structure.
678 *
679 * @returns VBox status code.
680 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
681 * case).
682 * @param pVmxTransient Pointer to the VMX transient structure.
683 */
684DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
685{
686 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
687 {
688 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
689 AssertRCReturn(rc, rc);
690 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
691 }
692 return VINF_SUCCESS;
693}
694
695
696/**
697 * Reads the IDT-vectoring information field from the VMCS into the VMX
698 * transient structure.
699 *
700 * @returns VBox status code.
701 * @param pVmxTransient Pointer to the VMX transient structure.
702 *
703 * @remarks No-long-jump zone!!!
704 */
705DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the IDT-vectoring error code from the VMCS into the VMX
719 * transient structure.
720 *
721 * @returns VBox status code.
722 * @param pVmxTransient Pointer to the VMX transient structure.
723 */
724DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
725{
726 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
727 {
728 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
729 AssertRCReturn(rc, rc);
730 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
731 }
732 return VINF_SUCCESS;
733}
734
735
736/**
737 * Enters VMX root mode operation on the current CPU.
738 *
739 * @returns VBox status code.
740 * @param pVM Pointer to the VM (optional, can be NULL, after
741 * a resume).
742 * @param HCPhysCpuPage Physical address of the VMXON region.
743 * @param pvCpuPage Pointer to the VMXON region.
744 */
745static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
746{
747 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
748 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
749 Assert(pvCpuPage);
750 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
751
752 if (pVM)
753 {
754 /* Write the VMCS revision dword to the VMXON region. */
755 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
756 }
757
758 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
759 RTCCUINTREG uEflags = ASMIntDisableFlags();
760
761 /* Enable the VMX bit in CR4 if necessary. */
762 RTCCUINTREG uCr4 = ASMGetCR4();
763 if (!(uCr4 & X86_CR4_VMXE))
764 ASMSetCR4(uCr4 | X86_CR4_VMXE);
765
766 /* Enter VMX root mode. */
767 int rc = VMXEnable(HCPhysCpuPage);
768 if (RT_FAILURE(rc))
769 ASMSetCR4(uCr4);
770
771 /* Restore interrupts. */
772 ASMSetFlags(uEflags);
773 return rc;
774}
775
776
777/**
778 * Exits VMX root mode operation on the current CPU.
779 *
780 * @returns VBox status code.
781 */
782static int hmR0VmxLeaveRootMode(void)
783{
784 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
785
786 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
787 RTCCUINTREG uEflags = ASMIntDisableFlags();
788
789 /* If we're for some reason not in VMX root mode, then don't leave it. */
790 RTCCUINTREG uHostCR4 = ASMGetCR4();
791
792 int rc;
793 if (uHostCR4 & X86_CR4_VMXE)
794 {
795 /* Exit VMX root mode and clear the VMX bit in CR4. */
796 VMXDisable();
797 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
798 rc = VINF_SUCCESS;
799 }
800 else
801 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
802
803 /* Restore interrupts. */
804 ASMSetFlags(uEflags);
805 return rc;
806}
807
808
809/**
810 * Allocates and maps one physically contiguous page. The allocated page is
811 * zero'd out. (Used by various VT-x structures).
812 *
813 * @returns IPRT status code.
814 * @param pMemObj Pointer to the ring-0 memory object.
815 * @param ppVirt Where to store the virtual address of the
816 * allocation.
817 * @param pPhys Where to store the physical address of the
818 * allocation.
819 */
820DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
821{
822 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
823 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
824 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
825
826 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
827 if (RT_FAILURE(rc))
828 return rc;
829 *ppVirt = RTR0MemObjAddress(*pMemObj);
830 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
831 ASMMemZero32(*ppVirt, PAGE_SIZE);
832 return VINF_SUCCESS;
833}
834
835
836/**
837 * Frees and unmaps an allocated physical page.
838 *
839 * @param pMemObj Pointer to the ring-0 memory object.
840 * @param ppVirt Where to re-initialize the virtual address of
841 * allocation as 0.
842 * @param pHCPhys Where to re-initialize the physical address of the
843 * allocation as 0.
844 */
845DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
846{
847 AssertPtr(pMemObj);
848 AssertPtr(ppVirt);
849 AssertPtr(pHCPhys);
850 if (*pMemObj != NIL_RTR0MEMOBJ)
851 {
852 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
853 AssertRC(rc);
854 *pMemObj = NIL_RTR0MEMOBJ;
855 *ppVirt = 0;
856 *pHCPhys = 0;
857 }
858}
859
860
861/**
862 * Worker function to free VT-x related structures.
863 *
864 * @returns IPRT status code.
865 * @param pVM Pointer to the VM.
866 */
867static void hmR0VmxStructsFree(PVM pVM)
868{
869 for (VMCPUID i = 0; i < pVM->cCpus; i++)
870 {
871 PVMCPU pVCpu = &pVM->aCpus[i];
872 AssertPtr(pVCpu);
873
874 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
875 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
876
877 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
879
880 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
882 }
883
884 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
885#ifdef VBOX_WITH_CRASHDUMP_MAGIC
886 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
887#endif
888}
889
890
891/**
892 * Worker function to allocate VT-x related VM structures.
893 *
894 * @returns IPRT status code.
895 * @param pVM Pointer to the VM.
896 */
897static int hmR0VmxStructsAlloc(PVM pVM)
898{
899 /*
900 * Initialize members up-front so we can cleanup properly on allocation failure.
901 */
902#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
903 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
904 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
905 pVM->hm.s.vmx.HCPhys##a_Name = 0;
906
907#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
908 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
909 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
910 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
911
912#ifdef VBOX_WITH_CRASHDUMP_MAGIC
913 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
914#endif
915 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
916
917 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
918 for (VMCPUID i = 0; i < pVM->cCpus; i++)
919 {
920 PVMCPU pVCpu = &pVM->aCpus[i];
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
926 }
927#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
928#undef VMXLOCAL_INIT_VM_MEMOBJ
929
930 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
931 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
932 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
933 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
934
935 /*
936 * Allocate all the VT-x structures.
937 */
938 int rc = VINF_SUCCESS;
939#ifdef VBOX_WITH_CRASHDUMP_MAGIC
940 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
941 if (RT_FAILURE(rc))
942 goto cleanup;
943 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
944 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
945#endif
946
947 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
948 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
949 {
950 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
951 &pVM->hm.s.vmx.HCPhysApicAccess);
952 if (RT_FAILURE(rc))
953 goto cleanup;
954 }
955
956 /*
957 * Initialize per-VCPU VT-x structures.
958 */
959 for (VMCPUID i = 0; i < pVM->cCpus; i++)
960 {
961 PVMCPU pVCpu = &pVM->aCpus[i];
962 AssertPtr(pVCpu);
963
964 /* Allocate the VM control structure (VMCS). */
965 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
966 if (RT_FAILURE(rc))
967 goto cleanup;
968
969 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
970 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
971 {
972 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
973 &pVCpu->hm.s.vmx.HCPhysVirtApic);
974 if (RT_FAILURE(rc))
975 goto cleanup;
976 }
977
978 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
979 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
980 {
981 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
982 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
983 if (RT_FAILURE(rc))
984 goto cleanup;
985 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
986 }
987
988 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
989 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
990 if (RT_FAILURE(rc))
991 goto cleanup;
992
993 /* Allocate the VM-exit MSR-load page for the host MSRs. */
994 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
995 if (RT_FAILURE(rc))
996 goto cleanup;
997 }
998
999 return VINF_SUCCESS;
1000
1001cleanup:
1002 hmR0VmxStructsFree(pVM);
1003 return rc;
1004}
1005
1006
1007/**
1008 * Does global VT-x initialization (called during module initialization).
1009 *
1010 * @returns VBox status code.
1011 */
1012VMMR0DECL(int) VMXR0GlobalInit(void)
1013{
1014#ifdef HMVMX_USE_FUNCTION_TABLE
1015 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1016# ifdef VBOX_STRICT
1017 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1018 Assert(g_apfnVMExitHandlers[i]);
1019# endif
1020#endif
1021 return VINF_SUCCESS;
1022}
1023
1024
1025/**
1026 * Does global VT-x termination (called during module termination).
1027 */
1028VMMR0DECL(void) VMXR0GlobalTerm()
1029{
1030 /* Nothing to do currently. */
1031}
1032
1033
1034/**
1035 * Sets up and activates VT-x on the current CPU.
1036 *
1037 * @returns VBox status code.
1038 * @param pCpu Pointer to the global CPU info struct.
1039 * @param pVM Pointer to the VM (can be NULL after a host resume
1040 * operation).
1041 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1042 * fEnabledByHost is true).
1043 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1044 * @a fEnabledByHost is true).
1045 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1046 * enable VT-x on the host.
1047 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1048 */
1049VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1050 void *pvMsrs)
1051{
1052 Assert(pCpu);
1053 Assert(pvMsrs);
1054 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1055
1056 /* Enable VT-x if it's not already enabled by the host. */
1057 if (!fEnabledByHost)
1058 {
1059 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1060 if (RT_FAILURE(rc))
1061 return rc;
1062 }
1063
1064 /*
1065 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1066 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1067 */
1068 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1069 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1070 {
1071 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1072 pCpu->fFlushAsidBeforeUse = false;
1073 }
1074 else
1075 pCpu->fFlushAsidBeforeUse = true;
1076
1077 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1078 ++pCpu->cTlbFlushes;
1079
1080 return VINF_SUCCESS;
1081}
1082
1083
1084/**
1085 * Deactivates VT-x on the current CPU.
1086 *
1087 * @returns VBox status code.
1088 * @param pCpu Pointer to the global CPU info struct.
1089 * @param pvCpuPage Pointer to the VMXON region.
1090 * @param HCPhysCpuPage Physical address of the VMXON region.
1091 *
1092 * @remarks This function should never be called when SUPR0EnableVTx() or
1093 * similar was used to enable VT-x on the host.
1094 */
1095VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1096{
1097 NOREF(pCpu);
1098 NOREF(pvCpuPage);
1099 NOREF(HCPhysCpuPage);
1100
1101 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1102 return hmR0VmxLeaveRootMode();
1103}
1104
1105
1106/**
1107 * Sets the permission bits for the specified MSR in the MSR bitmap.
1108 *
1109 * @param pVCpu Pointer to the VMCPU.
1110 * @param uMSR The MSR value.
1111 * @param enmRead Whether reading this MSR causes a VM-exit.
1112 * @param enmWrite Whether writing this MSR causes a VM-exit.
1113 */
1114static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1115{
1116 int32_t iBit;
1117 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1118
1119 /*
1120 * Layout:
1121 * 0x000 - 0x3ff - Low MSR read bits
1122 * 0x400 - 0x7ff - High MSR read bits
1123 * 0x800 - 0xbff - Low MSR write bits
1124 * 0xc00 - 0xfff - High MSR write bits
1125 */
1126 if (uMsr <= 0x00001FFF)
1127 iBit = uMsr;
1128 else if ( uMsr >= 0xC0000000
1129 && uMsr <= 0xC0001FFF)
1130 {
1131 iBit = (uMsr - 0xC0000000);
1132 pbMsrBitmap += 0x400;
1133 }
1134 else
1135 {
1136 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1137 return;
1138 }
1139
1140 Assert(iBit <= 0x1fff);
1141 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1142 ASMBitSet(pbMsrBitmap, iBit);
1143 else
1144 ASMBitClear(pbMsrBitmap, iBit);
1145
1146 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1147 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1148 else
1149 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1150}
1151
1152
1153#ifdef VBOX_STRICT
1154/**
1155 * Gets the permission bits for the specified MSR in the MSR bitmap.
1156 *
1157 * @returns VBox status code.
1158 * @retval VINF_SUCCESS if the specified MSR is found.
1159 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1160 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1161 *
1162 * @param pVCpu Pointer to the VMCPU.
1163 * @param uMsr The MSR.
1164 * @param penmRead Where to store the read permissions.
1165 * @param penmWrite Where to store the write permissions.
1166 */
1167static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1168{
1169 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1170 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1171 int32_t iBit;
1172 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1173
1174 /* See hmR0VmxSetMsrPermission() for the layout. */
1175 if (uMsr <= 0x00001FFF)
1176 iBit = uMsr;
1177 else if ( uMsr >= 0xC0000000
1178 && uMsr <= 0xC0001FFF)
1179 {
1180 iBit = (uMsr - 0xC0000000);
1181 pbMsrBitmap += 0x400;
1182 }
1183 else
1184 {
1185 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1186 return VERR_NOT_SUPPORTED;
1187 }
1188
1189 Assert(iBit <= 0x1fff);
1190 if (ASMBitTest(pbMsrBitmap, iBit))
1191 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1192 else
1193 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1194
1195 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1196 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1197 else
1198 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1199 return VINF_SUCCESS;
1200}
1201#endif /* VBOX_STRICT */
1202
1203
1204/**
1205 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1206 * area.
1207 *
1208 * @returns VBox status code.
1209 * @param pVCpu Pointer to the VMCPU.
1210 * @param cMsrs The number of MSRs.
1211 */
1212DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1213{
1214 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1215 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1216 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1217 {
1218 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1219 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1221 }
1222
1223 /* Update number of guest MSRs to load/store across the world-switch. */
1224 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1228 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1229
1230 /* Update the VCPU's copy of the MSR count. */
1231 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1232
1233 return VINF_SUCCESS;
1234}
1235
1236
1237/**
1238 * Adds a new (or updates the value of an existing) guest/host MSR
1239 * pair to be swapped during the world-switch as part of the
1240 * auto-load/store MSR area in the VMCS.
1241 *
1242 * @returns true if the MSR was added -and- its value was updated, false
1243 * otherwise.
1244 * @param pVCpu Pointer to the VMCPU.
1245 * @param uMsr The MSR.
1246 * @param uGuestMsr Value of the guest MSR.
1247 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1248 * necessary.
1249 */
1250static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1251{
1252 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1253 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1254 uint32_t i;
1255 for (i = 0; i < cMsrs; i++)
1256 {
1257 if (pGuestMsr->u32Msr == uMsr)
1258 break;
1259 pGuestMsr++;
1260 }
1261
1262 bool fAdded = false;
1263 if (i == cMsrs)
1264 {
1265 ++cMsrs;
1266 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1267 AssertRC(rc);
1268
1269 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1270 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1271 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1272
1273 fAdded = true;
1274 }
1275
1276 /* Update the MSR values in the auto-load/store MSR area. */
1277 pGuestMsr->u32Msr = uMsr;
1278 pGuestMsr->u64Value = uGuestMsrValue;
1279
1280 /* Create/update the MSR slot in the host MSR area. */
1281 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1282 pHostMsr += i;
1283 pHostMsr->u32Msr = uMsr;
1284
1285 /*
1286 * Update the host MSR only when requested by the caller AND when we're
1287 * adding it to the auto-load/store area. Otherwise, it would have been
1288 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1289 */
1290 bool fUpdatedMsrValue = false;
1291 if ( fAdded
1292 && fUpdateHostMsr)
1293 {
1294 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1295 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1296 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1297 fUpdatedMsrValue = true;
1298 }
1299
1300 return fUpdatedMsrValue;
1301}
1302
1303
1304/**
1305 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1306 * auto-load/store MSR area in the VMCS.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1316 for (uint32_t i = 0; i < cMsrs; i++)
1317 {
1318 /* Find the MSR. */
1319 if (pGuestMsr->u32Msr == uMsr)
1320 {
1321 /* If it's the last MSR, simply reduce the count. */
1322 if (i == cMsrs - 1)
1323 {
1324 --cMsrs;
1325 break;
1326 }
1327
1328 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1329 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 pLastGuestMsr += cMsrs - 1;
1331 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1332 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1333
1334 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1336 pLastHostMsr += cMsrs - 1;
1337 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1338 pHostMsr->u64Value = pLastHostMsr->u64Value;
1339 --cMsrs;
1340 break;
1341 }
1342 pGuestMsr++;
1343 }
1344
1345 /* Update the VMCS if the count changed (meaning the MSR was found). */
1346 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1347 {
1348 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1349 AssertRCReturn(rc, rc);
1350
1351 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1352 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1353 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1354
1355 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1356 return VINF_SUCCESS;
1357 }
1358
1359 return VERR_NOT_FOUND;
1360}
1361
1362
1363/**
1364 * Checks if the specified guest MSR is part of the auto-load/store area in
1365 * the VMCS.
1366 *
1367 * @returns true if found, false otherwise.
1368 * @param pVCpu Pointer to the VMCPU.
1369 * @param uMsr The MSR to find.
1370 */
1371static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1372{
1373 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1375
1376 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1377 {
1378 if (pGuestMsr->u32Msr == uMsr)
1379 return true;
1380 }
1381 return false;
1382}
1383
1384
1385/**
1386 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1387 *
1388 * @param pVCpu Pointer to the VMCPU.
1389 *
1390 * @remarks No-long-jump zone!!!
1391 */
1392static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1393{
1394 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1395 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1396 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1397 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1398
1399 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1400 {
1401 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1402
1403 /*
1404 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1405 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1406 */
1407 if (pHostMsr->u32Msr == MSR_K6_EFER)
1408 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1409 else
1410 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1411 }
1412
1413 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1414}
1415
1416
1417#if HC_ARCH_BITS == 64
1418/**
1419 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1420 * perform lazy restoration of the host MSRs while leaving VT-x.
1421 *
1422 * @param pVCpu Pointer to the VMCPU.
1423 *
1424 * @remarks No-long-jump zone!!!
1425 */
1426static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1427{
1428 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1429
1430 /*
1431 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1432 */
1433 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1434 {
1435 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1436 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1437 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1438 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1439 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1440 }
1441}
1442
1443
1444/**
1445 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1446 * lazily while leaving VT-x.
1447 *
1448 * @returns true if it does, false otherwise.
1449 * @param pVCpu Pointer to the VMCPU.
1450 * @param uMsr The MSR to check.
1451 */
1452static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1453{
1454 NOREF(pVCpu);
1455 switch (uMsr)
1456 {
1457 case MSR_K8_LSTAR:
1458 case MSR_K6_STAR:
1459 case MSR_K8_SF_MASK:
1460 case MSR_K8_KERNEL_GS_BASE:
1461 return true;
1462 }
1463 return false;
1464}
1465
1466
1467/**
1468 * Saves a set of guests MSRs back into the guest-CPU context.
1469 *
1470 * @param pVCpu Pointer to the VMCPU.
1471 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1472 * out-of-sync. Make sure to update the required fields
1473 * before using them.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1481
1482 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1483 {
1484 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1485 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1486 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1487 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1488 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1489 }
1490}
1491
1492
1493/**
1494 * Loads a set of guests MSRs to allow read/passthru to the guest.
1495 *
1496 * The name of this function is slightly confusing. This function does NOT
1497 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1498 * common prefix for functions dealing with "lazy restoration" of the shared
1499 * MSRs.
1500 *
1501 * @param pVCpu Pointer to the VMCPU.
1502 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1503 * out-of-sync. Make sure to update the required fields
1504 * before using them.
1505 *
1506 * @remarks No-long-jump zone!!!
1507 */
1508static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1509{
1510 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1511 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1512
1513 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1514 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1515 {
1516#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1517 do { \
1518 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1519 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1520 else \
1521 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1522 } while (0)
1523
1524 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1525 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1526 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1527 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1528#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1529 }
1530 else
1531 {
1532 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1533 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1534 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1535 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1536 }
1537 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1538}
1539
1540
1541/**
1542 * Performs lazy restoration of the set of host MSRs if they were previously
1543 * loaded with guest MSR values.
1544 *
1545 * @param pVCpu Pointer to the VMCPU.
1546 *
1547 * @remarks No-long-jump zone!!!
1548 * @remarks The guest MSRs should have been saved back into the guest-CPU
1549 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1550 */
1551static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1552{
1553 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1554 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1555
1556 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1557 {
1558 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1559 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1560 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1561 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1562 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1563 }
1564 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1565}
1566#endif /* HC_ARCH_BITS == 64 */
1567
1568
1569/**
1570 * Verifies that our cached values of the VMCS controls are all
1571 * consistent with what's actually present in the VMCS.
1572 *
1573 * @returns VBox status code.
1574 * @param pVCpu Pointer to the VMCPU.
1575 */
1576static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1577{
1578 uint32_t u32Val;
1579 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1580 AssertRCReturn(rc, rc);
1581 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1582 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1583
1584 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1585 AssertRCReturn(rc, rc);
1586 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1587 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1588
1589 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1590 AssertRCReturn(rc, rc);
1591 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1592 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1593
1594 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1595 AssertRCReturn(rc, rc);
1596 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1597 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1598
1599 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1600 AssertRCReturn(rc, rc);
1601 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1602 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1603
1604 return VINF_SUCCESS;
1605}
1606
1607
1608#ifdef VBOX_STRICT
1609/**
1610 * Verifies that our cached host EFER value has not changed
1611 * since we cached it.
1612 *
1613 * @param pVCpu Pointer to the VMCPU.
1614 */
1615static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1616{
1617 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1618
1619 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1620 {
1621 uint64_t u64Val;
1622 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1623 AssertRC(rc);
1624
1625 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1626 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1627 }
1628}
1629
1630
1631/**
1632 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1633 * VMCS are correct.
1634 *
1635 * @param pVCpu Pointer to the VMCPU.
1636 */
1637static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1638{
1639 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1640
1641 /* Verify MSR counts in the VMCS are what we think it should be. */
1642 uint32_t cMsrs;
1643 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1644 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1647 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1648
1649 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1650 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1651
1652 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1653 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1654 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1655 {
1656 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1657 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1658 pGuestMsr->u32Msr, cMsrs));
1659
1660 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1661 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1662 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1663
1664 /* Verify that the permissions are as expected in the MSR bitmap. */
1665 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1666 {
1667 VMXMSREXITREAD enmRead;
1668 VMXMSREXITWRITE enmWrite;
1669 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1670 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1671 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1672 {
1673 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1674 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1675 }
1676 else
1677 {
1678 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1679 pGuestMsr->u32Msr, cMsrs));
1680 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1681 pGuestMsr->u32Msr, cMsrs));
1682 }
1683 }
1684 }
1685}
1686#endif /* VBOX_STRICT */
1687
1688
1689/**
1690 * Flushes the TLB using EPT.
1691 *
1692 * @returns VBox status code.
1693 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1694 * enmFlush).
1695 * @param enmFlush Type of flush.
1696 *
1697 * @remarks Caller is responsible for making sure this function is called only
1698 * when NestedPaging is supported and providing @a enmFlush that is
1699 * supported by the CPU.
1700 * @remarks Can be called with interrupts disabled.
1701 */
1702static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1703{
1704 uint64_t au64Descriptor[2];
1705 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1706 au64Descriptor[0] = 0;
1707 else
1708 {
1709 Assert(pVCpu);
1710 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1711 }
1712 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1713
1714 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1715 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1716 rc));
1717 if ( RT_SUCCESS(rc)
1718 && pVCpu)
1719 {
1720 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1721 }
1722}
1723
1724
1725/**
1726 * Flushes the TLB using VPID.
1727 *
1728 * @returns VBox status code.
1729 * @param pVM Pointer to the VM.
1730 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1731 * enmFlush).
1732 * @param enmFlush Type of flush.
1733 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1734 * on @a enmFlush).
1735 *
1736 * @remarks Can be called with interrupts disabled.
1737 */
1738static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1739{
1740 NOREF(pVM);
1741 AssertPtr(pVM);
1742 Assert(pVM->hm.s.vmx.fVpid);
1743
1744 uint64_t au64Descriptor[2];
1745 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1746 {
1747 au64Descriptor[0] = 0;
1748 au64Descriptor[1] = 0;
1749 }
1750 else
1751 {
1752 AssertPtr(pVCpu);
1753 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1754 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1755 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1756 au64Descriptor[1] = GCPtr;
1757 }
1758
1759 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1760 AssertMsg(rc == VINF_SUCCESS,
1761 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1762 if ( RT_SUCCESS(rc)
1763 && pVCpu)
1764 {
1765 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1766 }
1767}
1768
1769
1770/**
1771 * Invalidates a guest page by guest virtual address. Only relevant for
1772 * EPT/VPID, otherwise there is nothing really to invalidate.
1773 *
1774 * @returns VBox status code.
1775 * @param pVM Pointer to the VM.
1776 * @param pVCpu Pointer to the VMCPU.
1777 * @param GCVirt Guest virtual address of the page to invalidate.
1778 */
1779VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1780{
1781 AssertPtr(pVM);
1782 AssertPtr(pVCpu);
1783 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1784
1785 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1786 if (!fFlushPending)
1787 {
1788 /*
1789 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1790 * See @bugref{6043} and @bugref{6177}.
1791 *
1792 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1793 * function maybe called in a loop with individual addresses.
1794 */
1795 if (pVM->hm.s.vmx.fVpid)
1796 {
1797 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1798 {
1799 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1800 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1801 }
1802 else
1803 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1804 }
1805 else if (pVM->hm.s.fNestedPaging)
1806 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1807 }
1808
1809 return VINF_SUCCESS;
1810}
1811
1812
1813/**
1814 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1815 * otherwise there is nothing really to invalidate.
1816 *
1817 * @returns VBox status code.
1818 * @param pVM Pointer to the VM.
1819 * @param pVCpu Pointer to the VMCPU.
1820 * @param GCPhys Guest physical address of the page to invalidate.
1821 */
1822VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1823{
1824 NOREF(pVM); NOREF(GCPhys);
1825 LogFlowFunc(("%RGp\n", GCPhys));
1826
1827 /*
1828 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1829 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1830 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1831 */
1832 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1833 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1834 return VINF_SUCCESS;
1835}
1836
1837
1838/**
1839 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1840 * case where neither EPT nor VPID is supported by the CPU.
1841 *
1842 * @param pVM Pointer to the VM.
1843 * @param pVCpu Pointer to the VMCPU.
1844 * @param pCpu Pointer to the global HM struct.
1845 *
1846 * @remarks Called with interrupts disabled.
1847 */
1848static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1849{
1850 AssertPtr(pVCpu);
1851 AssertPtr(pCpu);
1852 NOREF(pVM);
1853
1854 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1855
1856 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1857#if 0
1858 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1859 pVCpu->hm.s.TlbShootdown.cPages = 0;
1860#endif
1861
1862 Assert(pCpu->idCpu != NIL_RTCPUID);
1863 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1864 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1865 pVCpu->hm.s.fForceTLBFlush = false;
1866 return;
1867}
1868
1869
1870/**
1871 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1872 *
1873 * @param pVM Pointer to the VM.
1874 * @param pVCpu Pointer to the VMCPU.
1875 * @param pCpu Pointer to the global HM CPU struct.
1876 * @remarks All references to "ASID" in this function pertains to "VPID" in
1877 * Intel's nomenclature. The reason is, to avoid confusion in compare
1878 * statements since the host-CPU copies are named "ASID".
1879 *
1880 * @remarks Called with interrupts disabled.
1881 */
1882static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1883{
1884#ifdef VBOX_WITH_STATISTICS
1885 bool fTlbFlushed = false;
1886# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1887# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1888 if (!fTlbFlushed) \
1889 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1890 } while (0)
1891#else
1892# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1893# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1894#endif
1895
1896 AssertPtr(pVM);
1897 AssertPtr(pCpu);
1898 AssertPtr(pVCpu);
1899 Assert(pCpu->idCpu != NIL_RTCPUID);
1900
1901 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1902 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1903 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1904
1905 /*
1906 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1907 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1908 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1909 */
1910 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1911 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1912 {
1913 ++pCpu->uCurrentAsid;
1914 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1915 {
1916 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1917 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1918 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1919 }
1920
1921 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924
1925 /*
1926 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1927 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1928 */
1929 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1930 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1931 HMVMX_SET_TAGGED_TLB_FLUSHED();
1932 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1933 }
1934
1935 /* Check for explicit TLB shootdowns. */
1936 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1937 {
1938 /*
1939 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1940 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1941 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1942 * but not guest-physical mappings.
1943 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1944 */
1945 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1946 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1947 HMVMX_SET_TAGGED_TLB_FLUSHED();
1948 }
1949
1950 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1951 * where it is commented out. Support individual entry flushing
1952 * someday. */
1953#if 0
1954 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1955 {
1956 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1957
1958 /*
1959 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1960 * as supported by the CPU.
1961 */
1962 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1963 {
1964 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1965 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1966 }
1967 else
1968 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1969
1970 HMVMX_SET_TAGGED_TLB_FLUSHED();
1971 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1972 pVCpu->hm.s.TlbShootdown.cPages = 0;
1973 }
1974#endif
1975
1976 pVCpu->hm.s.fForceTLBFlush = false;
1977
1978 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1979
1980 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1981 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1982 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1983 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1984 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1985 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1986 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1987 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1988 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1989
1990 /* Update VMCS with the VPID. */
1991 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1992 AssertRC(rc);
1993
1994#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1995}
1996
1997
1998/**
1999 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2000 *
2001 * @returns VBox status code.
2002 * @param pVM Pointer to the VM.
2003 * @param pVCpu Pointer to the VMCPU.
2004 * @param pCpu Pointer to the global HM CPU struct.
2005 *
2006 * @remarks Called with interrupts disabled.
2007 */
2008static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2009{
2010 AssertPtr(pVM);
2011 AssertPtr(pVCpu);
2012 AssertPtr(pCpu);
2013 Assert(pCpu->idCpu != NIL_RTCPUID);
2014 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2015 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2016
2017 /*
2018 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2019 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2020 */
2021 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2022 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2023 {
2024 pVCpu->hm.s.fForceTLBFlush = true;
2025 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2026 }
2027
2028 /* Check for explicit TLB shootdown flushes. */
2029 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2030 {
2031 pVCpu->hm.s.fForceTLBFlush = true;
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2033 }
2034
2035 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2036 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2037
2038 if (pVCpu->hm.s.fForceTLBFlush)
2039 {
2040 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2041 pVCpu->hm.s.fForceTLBFlush = false;
2042 }
2043 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2044 * where it is commented out. Support individual entry flushing
2045 * someday. */
2046#if 0
2047 else
2048 {
2049 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2050 {
2051 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2052 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2053 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2054 }
2055 else
2056 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2057
2058 pVCpu->hm.s.TlbShootdown.cPages = 0;
2059 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2060 }
2061#endif
2062}
2063
2064
2065/**
2066 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2067 *
2068 * @returns VBox status code.
2069 * @param pVM Pointer to the VM.
2070 * @param pVCpu Pointer to the VMCPU.
2071 * @param pCpu Pointer to the global HM CPU struct.
2072 *
2073 * @remarks Called with interrupts disabled.
2074 */
2075static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2076{
2077 AssertPtr(pVM);
2078 AssertPtr(pVCpu);
2079 AssertPtr(pCpu);
2080 Assert(pCpu->idCpu != NIL_RTCPUID);
2081 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2082 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2083
2084 /*
2085 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2086 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2087 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2088 */
2089 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2090 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2091 {
2092 pVCpu->hm.s.fForceTLBFlush = true;
2093 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2094 }
2095
2096 /* Check for explicit TLB shootdown flushes. */
2097 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2098 {
2099 /*
2100 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2101 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2102 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2103 */
2104 pVCpu->hm.s.fForceTLBFlush = true;
2105 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2106 }
2107
2108 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2109 if (pVCpu->hm.s.fForceTLBFlush)
2110 {
2111 ++pCpu->uCurrentAsid;
2112 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2113 {
2114 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2115 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2116 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2117 }
2118
2119 pVCpu->hm.s.fForceTLBFlush = false;
2120 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2121 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2122 if (pCpu->fFlushAsidBeforeUse)
2123 {
2124 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2125 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2126 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2127 {
2128 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2129 pCpu->fFlushAsidBeforeUse = false;
2130 }
2131 else
2132 {
2133 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2134 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2135 }
2136 }
2137 }
2138 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2139 * where it is commented out. Support individual entry flushing
2140 * someday. */
2141#if 0
2142 else
2143 {
2144 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2145 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2146 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2147 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2148
2149 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2150 {
2151 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2152 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2153 {
2154 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2155 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2156 }
2157 else
2158 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2159
2160 pVCpu->hm.s.TlbShootdown.cPages = 0;
2161 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2162 }
2163 else
2164 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2165 }
2166#endif
2167
2168 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2169 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2170 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2171 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2172 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2173 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2174 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2175
2176 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2177 AssertRC(rc);
2178}
2179
2180
2181/**
2182 * Flushes the guest TLB entry based on CPU capabilities.
2183 *
2184 * @param pVCpu Pointer to the VMCPU.
2185 * @param pCpu Pointer to the global HM CPU struct.
2186 */
2187DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2188{
2189#ifdef HMVMX_ALWAYS_FLUSH_TLB
2190 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2191#endif
2192 PVM pVM = pVCpu->CTX_SUFF(pVM);
2193 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2194 {
2195 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2196 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2197 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2198 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2199 default:
2200 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2201 break;
2202 }
2203
2204 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2205 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2206
2207 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2208}
2209
2210
2211/**
2212 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2213 * TLB entries from the host TLB before VM-entry.
2214 *
2215 * @returns VBox status code.
2216 * @param pVM Pointer to the VM.
2217 */
2218static int hmR0VmxSetupTaggedTlb(PVM pVM)
2219{
2220 /*
2221 * Determine optimal flush type for Nested Paging.
2222 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2223 * guest execution (see hmR3InitFinalizeR0()).
2224 */
2225 if (pVM->hm.s.fNestedPaging)
2226 {
2227 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2228 {
2229 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2230 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2231 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2232 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2233 else
2234 {
2235 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2236 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2237 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2238 }
2239
2240 /* Make sure the write-back cacheable memory type for EPT is supported. */
2241 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2242 {
2243 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2244 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2245 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2246 }
2247 }
2248 else
2249 {
2250 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2251 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254 }
2255
2256 /*
2257 * Determine optimal flush type for VPID.
2258 */
2259 if (pVM->hm.s.vmx.fVpid)
2260 {
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2262 {
2263 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2264 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2265 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2266 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2267 else
2268 {
2269 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2270 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2271 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2272 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2273 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2274 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2275 pVM->hm.s.vmx.fVpid = false;
2276 }
2277 }
2278 else
2279 {
2280 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2281 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2282 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2283 pVM->hm.s.vmx.fVpid = false;
2284 }
2285 }
2286
2287 /*
2288 * Setup the handler for flushing tagged-TLBs.
2289 */
2290 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2291 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2292 else if (pVM->hm.s.fNestedPaging)
2293 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2294 else if (pVM->hm.s.vmx.fVpid)
2295 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2296 else
2297 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2298 return VINF_SUCCESS;
2299}
2300
2301
2302/**
2303 * Sets up pin-based VM-execution controls in the VMCS.
2304 *
2305 * @returns VBox status code.
2306 * @param pVM Pointer to the VM.
2307 * @param pVCpu Pointer to the VMCPU.
2308 */
2309static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2310{
2311 AssertPtr(pVM);
2312 AssertPtr(pVCpu);
2313
2314 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2315 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2316
2317 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2318 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2319 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2320
2321 /* Enable the VMX preemption timer. */
2322 if (pVM->hm.s.vmx.fUsePreemptTimer)
2323 {
2324 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2325 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2326 }
2327
2328 if ((val & zap) != val)
2329 {
2330 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2331 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2332 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2333 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2334 }
2335
2336 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2337 AssertRCReturn(rc, rc);
2338
2339 /* Update VCPU with the currently set pin-based VM-execution controls. */
2340 pVCpu->hm.s.vmx.u32PinCtls = val;
2341 return rc;
2342}
2343
2344
2345/**
2346 * Sets up processor-based VM-execution controls in the VMCS.
2347 *
2348 * @returns VBox status code.
2349 * @param pVM Pointer to the VM.
2350 * @param pVMCPU Pointer to the VMCPU.
2351 */
2352static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2353{
2354 AssertPtr(pVM);
2355 AssertPtr(pVCpu);
2356
2357 int rc = VERR_INTERNAL_ERROR_5;
2358 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2359 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2360
2361 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2362 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2363 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2364 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2365 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2366 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2367 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2368
2369 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2370 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2371 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2372 {
2373 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2374 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2375 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2376 }
2377
2378 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2379 if (!pVM->hm.s.fNestedPaging)
2380 {
2381 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2382 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2383 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2384 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2385 }
2386
2387 /* Use TPR shadowing if supported by the CPU. */
2388 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2389 {
2390 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2391 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2392 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2393 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2394 AssertRCReturn(rc, rc);
2395
2396 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2397 /* CR8 writes causes a VM-exit based on TPR threshold. */
2398 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2399 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2400 }
2401 else
2402 {
2403 /*
2404 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2405 * Set this control only for 64-bit guests.
2406 */
2407 if (pVM->hm.s.fAllow64BitGuests)
2408 {
2409 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2410 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2411 }
2412 }
2413
2414 /* Use MSR-bitmaps if supported by the CPU. */
2415 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2416 {
2417 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2418
2419 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2420 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2421 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2422 AssertRCReturn(rc, rc);
2423
2424 /*
2425 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2426 * automatically as dedicated fields in the VMCS.
2427 */
2428 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2429 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2430 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2431 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2432 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2433
2434#if HC_ARCH_BITS == 64
2435 /*
2436 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2437 */
2438 if (pVM->hm.s.fAllow64BitGuests)
2439 {
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2443 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2444 }
2445#endif
2446 }
2447
2448 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2449 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2450 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2451
2452 if ((val & zap) != val)
2453 {
2454 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2455 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2456 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2457 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2458 }
2459
2460 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2461 AssertRCReturn(rc, rc);
2462
2463 /* Update VCPU with the currently set processor-based VM-execution controls. */
2464 pVCpu->hm.s.vmx.u32ProcCtls = val;
2465
2466 /*
2467 * Secondary processor-based VM-execution controls.
2468 */
2469 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2470 {
2471 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2472 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2473
2474 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2475 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2476
2477 if (pVM->hm.s.fNestedPaging)
2478 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2479 else
2480 {
2481 /*
2482 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2483 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2484 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2485 */
2486 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2487 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2488 }
2489
2490 if (pVM->hm.s.vmx.fVpid)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2492
2493 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2495
2496 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2497 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2498 * done dynamically. */
2499 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2500 {
2501 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2502 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2504 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2505 AssertRCReturn(rc, rc);
2506 }
2507
2508 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2509 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2510
2511 if ((val & zap) != val)
2512 {
2513 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2514 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2515 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2516 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2517 }
2518
2519 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2520 AssertRCReturn(rc, rc);
2521
2522 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2523 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2524 }
2525 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2526 {
2527 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2528 "available\n"));
2529 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2530 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2531 }
2532
2533 return VINF_SUCCESS;
2534}
2535
2536
2537/**
2538 * Sets up miscellaneous (everything other than Pin & Processor-based
2539 * VM-execution) control fields in the VMCS.
2540 *
2541 * @returns VBox status code.
2542 * @param pVM Pointer to the VM.
2543 * @param pVCpu Pointer to the VMCPU.
2544 */
2545static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2546{
2547 NOREF(pVM);
2548 AssertPtr(pVM);
2549 AssertPtr(pVCpu);
2550
2551 int rc = VERR_GENERAL_FAILURE;
2552
2553 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2554#if 0
2555 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2556 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2557 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2558
2559 /*
2560 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2561 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2562 * We thus use the exception bitmap to control it rather than use both.
2563 */
2564 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2565 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2566
2567 /** @todo Explore possibility of using IO-bitmaps. */
2568 /* All IO & IOIO instructions cause VM-exits. */
2569 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2570 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2571
2572 /* Initialize the MSR-bitmap area. */
2573 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2574 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2575 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2576#endif
2577
2578 /* Setup MSR auto-load/store area. */
2579 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2580 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2582 AssertRCReturn(rc, rc);
2583 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2584 AssertRCReturn(rc, rc);
2585
2586 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2587 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2588 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2589 AssertRCReturn(rc, rc);
2590
2591 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2592 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2593 AssertRCReturn(rc, rc);
2594
2595 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2596#if 0
2597 /* Setup debug controls */
2598 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2599 AssertRCReturn(rc, rc);
2600 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2601 AssertRCReturn(rc, rc);
2602#endif
2603
2604 return rc;
2605}
2606
2607
2608/**
2609 * Sets up the initial exception bitmap in the VMCS based on static conditions
2610 * (i.e. conditions that cannot ever change after starting the VM).
2611 *
2612 * @returns VBox status code.
2613 * @param pVM Pointer to the VM.
2614 * @param pVCpu Pointer to the VMCPU.
2615 */
2616static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2617{
2618 AssertPtr(pVM);
2619 AssertPtr(pVCpu);
2620
2621 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2622
2623 uint32_t u32XcptBitmap = 0;
2624
2625 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2626 if (!pVM->hm.s.fNestedPaging)
2627 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2628
2629 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2630 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2631 AssertRCReturn(rc, rc);
2632 return rc;
2633}
2634
2635
2636/**
2637 * Sets up the initial guest-state mask. The guest-state mask is consulted
2638 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2639 * for the nested virtualization case (as it would cause a VM-exit).
2640 *
2641 * @param pVCpu Pointer to the VMCPU.
2642 */
2643static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2644{
2645 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2646 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2647 return VINF_SUCCESS;
2648}
2649
2650
2651/**
2652 * Does per-VM VT-x initialization.
2653 *
2654 * @returns VBox status code.
2655 * @param pVM Pointer to the VM.
2656 */
2657VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2658{
2659 LogFlowFunc(("pVM=%p\n", pVM));
2660
2661 int rc = hmR0VmxStructsAlloc(pVM);
2662 if (RT_FAILURE(rc))
2663 {
2664 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2665 return rc;
2666 }
2667
2668 return VINF_SUCCESS;
2669}
2670
2671
2672/**
2673 * Does per-VM VT-x termination.
2674 *
2675 * @returns VBox status code.
2676 * @param pVM Pointer to the VM.
2677 */
2678VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2679{
2680 LogFlowFunc(("pVM=%p\n", pVM));
2681
2682#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2683 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2684 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2685#endif
2686 hmR0VmxStructsFree(pVM);
2687 return VINF_SUCCESS;
2688}
2689
2690
2691/**
2692 * Sets up the VM for execution under VT-x.
2693 * This function is only called once per-VM during initialization.
2694 *
2695 * @returns VBox status code.
2696 * @param pVM Pointer to the VM.
2697 */
2698VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2699{
2700 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2701 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2702
2703 LogFlowFunc(("pVM=%p\n", pVM));
2704
2705 /*
2706 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2707 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2708 */
2709 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2710 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2711 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2712 || !pVM->hm.s.vmx.pRealModeTSS))
2713 {
2714 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2715 return VERR_INTERNAL_ERROR;
2716 }
2717
2718#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2719 /*
2720 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2721 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2722 */
2723 if ( pVM->hm.s.fAllow64BitGuests
2724 && !HMVMX_IS_64BIT_HOST_MODE())
2725 {
2726 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2727 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2728 }
2729#endif
2730
2731 /* Initialize these always, see hmR3InitFinalizeR0().*/
2732 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2733 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2734
2735 /* Setup the tagged-TLB flush handlers. */
2736 int rc = hmR0VmxSetupTaggedTlb(pVM);
2737 if (RT_FAILURE(rc))
2738 {
2739 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2740 return rc;
2741 }
2742
2743 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2744 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2745#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2746 if ( HMVMX_IS_64BIT_HOST_MODE()
2747 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2748 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2749 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2750 {
2751 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2752 }
2753#endif
2754
2755 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2756 {
2757 PVMCPU pVCpu = &pVM->aCpus[i];
2758 AssertPtr(pVCpu);
2759 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2760
2761 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2762 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2763
2764 /* Set revision dword at the beginning of the VMCS structure. */
2765 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2766
2767 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2768 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 /* Load this VMCS as the current VMCS. */
2773 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2774 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2775 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2776
2777 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2779 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2780
2781 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2787 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2788
2789 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2794 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2795 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2796
2797#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2798 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2800 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2801#endif
2802
2803 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2804 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2806 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2807
2808 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2809
2810 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2811 }
2812
2813 return VINF_SUCCESS;
2814}
2815
2816
2817/**
2818 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2819 * the VMCS.
2820 *
2821 * @returns VBox status code.
2822 * @param pVM Pointer to the VM.
2823 * @param pVCpu Pointer to the VMCPU.
2824 */
2825DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2826{
2827 NOREF(pVM); NOREF(pVCpu);
2828
2829 RTCCUINTREG uReg = ASMGetCR0();
2830 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2831 AssertRCReturn(rc, rc);
2832
2833#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2834 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2835 if (HMVMX_IS_64BIT_HOST_MODE())
2836 {
2837 uint64_t uRegCR3 = HMR0Get64bitCR3();
2838 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2839 }
2840 else
2841#endif
2842 {
2843 uReg = ASMGetCR3();
2844 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2845 }
2846 AssertRCReturn(rc, rc);
2847
2848 uReg = ASMGetCR4();
2849 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2850 AssertRCReturn(rc, rc);
2851 return rc;
2852}
2853
2854
2855#if HC_ARCH_BITS == 64
2856/**
2857 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2858 * requirements. See hmR0VmxSaveHostSegmentRegs().
2859 */
2860# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2861 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2862 { \
2863 bool fValidSelector = true; \
2864 if ((selValue) & X86_SEL_LDT) \
2865 { \
2866 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2867 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2868 } \
2869 if (fValidSelector) \
2870 { \
2871 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2872 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2873 } \
2874 (selValue) = 0; \
2875 }
2876#endif
2877
2878
2879/**
2880 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2881 * the host-state area in the VMCS.
2882 *
2883 * @returns VBox status code.
2884 * @param pVM Pointer to the VM.
2885 * @param pVCpu Pointer to the VMCPU.
2886 */
2887DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2888{
2889 NOREF(pVM);
2890 int rc = VERR_INTERNAL_ERROR_5;
2891
2892#if HC_ARCH_BITS == 64
2893 /*
2894 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2895 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2896 */
2897 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2898 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2899#endif
2900
2901 /*
2902 * Host DS, ES, FS and GS segment registers.
2903 */
2904#if HC_ARCH_BITS == 64
2905 RTSEL uSelDS = ASMGetDS();
2906 RTSEL uSelES = ASMGetES();
2907 RTSEL uSelFS = ASMGetFS();
2908 RTSEL uSelGS = ASMGetGS();
2909#else
2910 RTSEL uSelDS = 0;
2911 RTSEL uSelES = 0;
2912 RTSEL uSelFS = 0;
2913 RTSEL uSelGS = 0;
2914#endif
2915
2916 /* Recalculate which host-state bits need to be manually restored. */
2917 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2918
2919 /*
2920 * Host CS and SS segment registers.
2921 */
2922#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2923 RTSEL uSelCS;
2924 RTSEL uSelSS;
2925 if (HMVMX_IS_64BIT_HOST_MODE())
2926 {
2927 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2928 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2929 }
2930 else
2931 {
2932 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2933 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2934 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2935 }
2936#else
2937 RTSEL uSelCS = ASMGetCS();
2938 RTSEL uSelSS = ASMGetSS();
2939#endif
2940
2941 /*
2942 * Host TR segment register.
2943 */
2944 RTSEL uSelTR = ASMGetTR();
2945
2946#if HC_ARCH_BITS == 64
2947 /*
2948 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2949 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2950 */
2951 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2952 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2953 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2954 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2955# undef VMXLOCAL_ADJUST_HOST_SEG
2956#endif
2957
2958 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2959 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2960 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2961 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2962 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2963 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2964 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2965 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2966 Assert(uSelCS);
2967 Assert(uSelTR);
2968
2969 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2970#if 0
2971 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2972 Assert(uSelSS != 0);
2973#endif
2974
2975 /* Write these host selector fields into the host-state area in the VMCS. */
2976 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2977 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2978#if HC_ARCH_BITS == 64
2979 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2980 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2981 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2982 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2983#endif
2984 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2985
2986 /*
2987 * Host GDTR and IDTR.
2988 */
2989 RTGDTR Gdtr;
2990 RT_ZERO(Gdtr);
2991#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2992 if (HMVMX_IS_64BIT_HOST_MODE())
2993 {
2994 X86XDTR64 Gdtr64;
2995 X86XDTR64 Idtr64;
2996 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2997 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2999
3000 Gdtr.cbGdt = Gdtr64.cb;
3001 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3002 }
3003 else
3004#endif
3005 {
3006 RTIDTR Idtr;
3007 ASMGetGDTR(&Gdtr);
3008 ASMGetIDTR(&Idtr);
3009 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3010 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3011
3012#if HC_ARCH_BITS == 64
3013 /*
3014 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3015 * maximum limit (0xffff) on every VM-exit.
3016 */
3017 if (Gdtr.cbGdt != 0xffff)
3018 {
3019 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3020 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3021 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3022 }
3023
3024 /*
3025 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3026 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3027 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3028 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3029 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3030 * hosts where we are pretty sure it won't cause trouble.
3031 */
3032# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3033 if (Idtr.cbIdt < 0x0fff)
3034# else
3035 if (Idtr.cbIdt != 0xffff)
3036# endif
3037 {
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3039 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3040 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3041 }
3042#endif
3043 }
3044
3045 /*
3046 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3047 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3048 */
3049 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3050 {
3051 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3052 return VERR_VMX_INVALID_HOST_STATE;
3053 }
3054
3055 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3056#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3057 if (HMVMX_IS_64BIT_HOST_MODE())
3058 {
3059 /* We need the 64-bit TR base for hybrid darwin. */
3060 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3061 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3062 }
3063 else
3064#endif
3065 {
3066 uintptr_t uTRBase;
3067#if HC_ARCH_BITS == 64
3068 uTRBase = X86DESC64_BASE(pDesc);
3069
3070 /*
3071 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3072 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3073 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3074 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3075 *
3076 * [1] See Intel spec. 3.5 "System Descriptor Types".
3077 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3078 */
3079 Assert(pDesc->System.u4Type == 11);
3080 if ( pDesc->System.u16LimitLow != 0x67
3081 || pDesc->System.u4LimitHigh)
3082 {
3083 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3084 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3085
3086 /* Store the GDTR here as we need it while restoring TR. */
3087 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3088 }
3089#else
3090 uTRBase = X86DESC_BASE(pDesc);
3091#endif
3092 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3093 }
3094 AssertRCReturn(rc, rc);
3095
3096 /*
3097 * Host FS base and GS base.
3098 */
3099#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3100 if (HMVMX_IS_64BIT_HOST_MODE())
3101 {
3102 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3103 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3104 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3105 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3106
3107# if HC_ARCH_BITS == 64
3108 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3109 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3110 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3111 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3112 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3113# endif
3114 }
3115#endif
3116 return rc;
3117}
3118
3119
3120/**
3121 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3122 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3123 * the host after every successful VM-exit.
3124 *
3125 * @returns VBox status code.
3126 * @param pVM Pointer to the VM.
3127 * @param pVCpu Pointer to the VMCPU.
3128 *
3129 * @remarks No-long-jump zone!!!
3130 */
3131DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3132{
3133 NOREF(pVM);
3134
3135 AssertPtr(pVCpu);
3136 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3137
3138 int rc = VINF_SUCCESS;
3139#if HC_ARCH_BITS == 64
3140 if (pVM->hm.s.fAllow64BitGuests)
3141 hmR0VmxLazySaveHostMsrs(pVCpu);
3142#endif
3143
3144 /*
3145 * Host Sysenter MSRs.
3146 */
3147 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3148 AssertRCReturn(rc, rc);
3149#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3150 if (HMVMX_IS_64BIT_HOST_MODE())
3151 {
3152 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3153 AssertRCReturn(rc, rc);
3154 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3155 }
3156 else
3157 {
3158 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3159 AssertRCReturn(rc, rc);
3160 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3161 }
3162#elif HC_ARCH_BITS == 32
3163 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3164 AssertRCReturn(rc, rc);
3165 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3166#else
3167 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3168 AssertRCReturn(rc, rc);
3169 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3170#endif
3171 AssertRCReturn(rc, rc);
3172
3173 /*
3174 * Host EFER MSR.
3175 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3176 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3177 */
3178 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3179 {
3180 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3181 AssertRCReturn(rc, rc);
3182 }
3183
3184 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3185 * hmR0VmxLoadGuestExitCtls() !! */
3186
3187 return rc;
3188}
3189
3190
3191/**
3192 * Figures out if we need to swap the EFER MSR which is
3193 * particularly expensive.
3194 *
3195 * We check all relevant bits. For now, that's everything
3196 * besides LMA/LME, as these two bits are handled by VM-entry,
3197 * see hmR0VmxLoadGuestExitCtls() and
3198 * hmR0VMxLoadGuestEntryCtls().
3199 *
3200 * @returns true if we need to load guest EFER, false otherwise.
3201 * @param pVCpu Pointer to the VMCPU.
3202 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3203 * out-of-sync. Make sure to update the required fields
3204 * before using them.
3205 *
3206 * @remarks Requires EFER, CR4.
3207 * @remarks No-long-jump zone!!!
3208 */
3209static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3210{
3211#ifdef HMVMX_ALWAYS_SWAP_EFER
3212 return true;
3213#endif
3214
3215#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3216 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3217 if (CPUMIsGuestInLongMode(pVCpu))
3218 return false;
3219#endif
3220
3221 PVM pVM = pVCpu->CTX_SUFF(pVM);
3222 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3223 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3224
3225 /*
3226 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3227 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3228 */
3229 if ( CPUMIsGuestInLongMode(pVCpu)
3230 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3231 {
3232 return true;
3233 }
3234
3235 /*
3236 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3237 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3238 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3239 */
3240 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3241 && (pMixedCtx->cr0 & X86_CR0_PG)
3242 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3243 {
3244 /* Assert that host is PAE capable. */
3245 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3246 return true;
3247 }
3248
3249 /** @todo Check the latest Intel spec. for any other bits,
3250 * like SMEP/SMAP? */
3251 return false;
3252}
3253
3254
3255/**
3256 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3257 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3258 * controls".
3259 *
3260 * @returns VBox status code.
3261 * @param pVCpu Pointer to the VMCPU.
3262 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3263 * out-of-sync. Make sure to update the required fields
3264 * before using them.
3265 *
3266 * @remarks Requires EFER.
3267 * @remarks No-long-jump zone!!!
3268 */
3269DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3270{
3271 int rc = VINF_SUCCESS;
3272 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3273 {
3274 PVM pVM = pVCpu->CTX_SUFF(pVM);
3275 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3276 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3277
3278 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3279 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3280
3281 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3282 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3283 {
3284 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3285 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3286 }
3287 else
3288 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3289
3290 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3291 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3292 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3293 {
3294 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3295 Log4(("Load: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3296 }
3297
3298 /*
3299 * The following should -not- be set (since we're not in SMM mode):
3300 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3301 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3302 */
3303
3304 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3305 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3306
3307 if ((val & zap) != val)
3308 {
3309 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3310 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3311 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3312 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3313 }
3314
3315 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3316 AssertRCReturn(rc, rc);
3317
3318 /* Update VCPU with the currently set VM-exit controls. */
3319 pVCpu->hm.s.vmx.u32EntryCtls = val;
3320 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3321 }
3322 return rc;
3323}
3324
3325
3326/**
3327 * Sets up the VM-exit controls in the VMCS.
3328 *
3329 * @returns VBox status code.
3330 * @param pVM Pointer to the VM.
3331 * @param pVCpu Pointer to the VMCPU.
3332 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3333 * out-of-sync. Make sure to update the required fields
3334 * before using them.
3335 *
3336 * @remarks Requires EFER.
3337 */
3338DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3339{
3340 NOREF(pMixedCtx);
3341
3342 int rc = VINF_SUCCESS;
3343 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3344 {
3345 PVM pVM = pVCpu->CTX_SUFF(pVM);
3346 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3347 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3348
3349 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3350 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3351
3352 /*
3353 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3354 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3355 */
3356#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3357 if (HMVMX_IS_64BIT_HOST_MODE())
3358 {
3359 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3360 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3361 }
3362 else
3363 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3364#else
3365 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3366 {
3367 /* The switcher returns to long mode, EFER is managed by the switcher. */
3368 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3369 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3370 }
3371 else
3372 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3373#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3374
3375 /* If the newer VMCS fields for managing EFER exists, use it. */
3376 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3377 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3378 {
3379 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3380 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3381 Log4(("Load: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3382 }
3383
3384 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3385 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3386
3387 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3388 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3389 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3390
3391 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3392 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3393
3394 if ((val & zap) != val)
3395 {
3396 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3397 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3398 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3399 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3400 }
3401
3402 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3403 AssertRCReturn(rc, rc);
3404
3405 /* Update VCPU with the currently set VM-exit controls. */
3406 pVCpu->hm.s.vmx.u32ExitCtls = val;
3407 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3408 }
3409 return rc;
3410}
3411
3412
3413/**
3414 * Loads the guest APIC and related state.
3415 *
3416 * @returns VBox status code.
3417 * @param pVM Pointer to the VM.
3418 * @param pVCpu Pointer to the VMCPU.
3419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3420 * out-of-sync. Make sure to update the required fields
3421 * before using them.
3422 */
3423DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3424{
3425 NOREF(pMixedCtx);
3426
3427 int rc = VINF_SUCCESS;
3428 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3429 {
3430 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3431 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3432 {
3433 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3434
3435 bool fPendingIntr = false;
3436 uint8_t u8Tpr = 0;
3437 uint8_t u8PendingIntr = 0;
3438 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3439 AssertRCReturn(rc, rc);
3440
3441 /*
3442 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3443 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3444 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3445 * the interrupt when we VM-exit for other reasons.
3446 */
3447 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3448 uint32_t u32TprThreshold = 0;
3449 if (fPendingIntr)
3450 {
3451 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3452 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3453 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3454 if (u8PendingPriority <= u8TprPriority)
3455 u32TprThreshold = u8PendingPriority;
3456 else
3457 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3458 }
3459 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3460
3461 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3462 AssertRCReturn(rc, rc);
3463 }
3464
3465 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3466 }
3467 return rc;
3468}
3469
3470
3471/**
3472 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3473 *
3474 * @returns Guest's interruptibility-state.
3475 * @param pVCpu Pointer to the VMCPU.
3476 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3477 * out-of-sync. Make sure to update the required fields
3478 * before using them.
3479 *
3480 * @remarks No-long-jump zone!!!
3481 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3482 */
3483DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3484{
3485 /*
3486 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3487 * inhibit interrupts or clear any existing interrupt-inhibition.
3488 */
3489 uint32_t uIntrState = 0;
3490 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3491 {
3492 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3493 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3494 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3495 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3496 {
3497 /*
3498 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3499 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3500 */
3501 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3502 }
3503 else if (pMixedCtx->eflags.Bits.u1IF)
3504 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3505 else
3506 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3507 }
3508 return uIntrState;
3509}
3510
3511
3512/**
3513 * Loads the guest's interruptibility-state into the guest-state area in the
3514 * VMCS.
3515 *
3516 * @returns VBox status code.
3517 * @param pVCpu Pointer to the VMCPU.
3518 * @param uIntrState The interruptibility-state to set.
3519 */
3520static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3521{
3522 NOREF(pVCpu);
3523 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3524 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3525 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3526 AssertRCReturn(rc, rc);
3527 return rc;
3528}
3529
3530
3531/**
3532 * Loads the guest's RIP into the guest-state area in the VMCS.
3533 *
3534 * @returns VBox status code.
3535 * @param pVCpu Pointer to the VMCPU.
3536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3537 * out-of-sync. Make sure to update the required fields
3538 * before using them.
3539 *
3540 * @remarks No-long-jump zone!!!
3541 */
3542static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3543{
3544 int rc = VINF_SUCCESS;
3545 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3546 {
3547 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3548 AssertRCReturn(rc, rc);
3549
3550 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3551 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3552 }
3553 return rc;
3554}
3555
3556
3557/**
3558 * Loads the guest's RSP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu Pointer to the VMCPU.
3562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3563 * out-of-sync. Make sure to update the required fields
3564 * before using them.
3565 *
3566 * @remarks No-long-jump zone!!!
3567 */
3568static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3569{
3570 int rc = VINF_SUCCESS;
3571 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3572 {
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3574 AssertRCReturn(rc, rc);
3575
3576 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3577 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu Pointer to the VMCPU.
3588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3589 * out-of-sync. Make sure to update the required fields
3590 * before using them.
3591 *
3592 * @remarks No-long-jump zone!!!
3593 */
3594static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3595{
3596 int rc = VINF_SUCCESS;
3597 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3598 {
3599 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3600 Let us assert it as such and use 32-bit VMWRITE. */
3601 Assert(!(pMixedCtx->rflags.u64 >> 32));
3602 X86EFLAGS Eflags = pMixedCtx->eflags;
3603 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3604 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3605
3606 /*
3607 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3608 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3609 */
3610 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3611 {
3612 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3613 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3614 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3615 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3616 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3617 }
3618
3619 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3620 AssertRCReturn(rc, rc);
3621
3622 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3623 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3624 }
3625 return rc;
3626}
3627
3628
3629/**
3630 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3631 *
3632 * @returns VBox status code.
3633 * @param pVCpu Pointer to the VMCPU.
3634 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3635 * out-of-sync. Make sure to update the required fields
3636 * before using them.
3637 *
3638 * @remarks No-long-jump zone!!!
3639 */
3640DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3641{
3642 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3643 AssertRCReturn(rc, rc);
3644 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3645 AssertRCReturn(rc, rc);
3646 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3647 AssertRCReturn(rc, rc);
3648 return rc;
3649}
3650
3651
3652/**
3653 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3654 * CR0 is partially shared with the host and we have to consider the FPU bits.
3655 *
3656 * @returns VBox status code.
3657 * @param pVM Pointer to the VM.
3658 * @param pVCpu Pointer to the VMCPU.
3659 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3660 * out-of-sync. Make sure to update the required fields
3661 * before using them.
3662 *
3663 * @remarks No-long-jump zone!!!
3664 */
3665static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3666{
3667 /*
3668 * Guest CR0.
3669 * Guest FPU.
3670 */
3671 int rc = VINF_SUCCESS;
3672 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3673 {
3674 Assert(!(pMixedCtx->cr0 >> 32));
3675 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3676 PVM pVM = pVCpu->CTX_SUFF(pVM);
3677
3678 /* The guest's view (read access) of its CR0 is unblemished. */
3679 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3680 AssertRCReturn(rc, rc);
3681 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3682
3683 /* Setup VT-x's view of the guest CR0. */
3684 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3685 if (pVM->hm.s.fNestedPaging)
3686 {
3687 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3688 {
3689 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3690 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3691 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3692 }
3693 else
3694 {
3695 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3696 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3697 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3698 }
3699
3700 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3701 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3702 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3703
3704 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3705 AssertRCReturn(rc, rc);
3706 }
3707 else
3708 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3709
3710 /*
3711 * Guest FPU bits.
3712 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3713 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3714 */
3715 u32GuestCR0 |= X86_CR0_NE;
3716 bool fInterceptNM = false;
3717 if (CPUMIsGuestFPUStateActive(pVCpu))
3718 {
3719 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3720 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3721 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3722 }
3723 else
3724 {
3725 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3726 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3727 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3728 }
3729
3730 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3731 bool fInterceptMF = false;
3732 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3733 fInterceptMF = true;
3734
3735 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3736 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3737 {
3738 Assert(PDMVmmDevHeapIsEnabled(pVM));
3739 Assert(pVM->hm.s.vmx.pRealModeTSS);
3740 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3741 fInterceptNM = true;
3742 fInterceptMF = true;
3743 }
3744 else
3745 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3746
3747 if (fInterceptNM)
3748 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3749 else
3750 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3751
3752 if (fInterceptMF)
3753 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3754 else
3755 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3756
3757 /* Additional intercepts for debugging, define these yourself explicitly. */
3758#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3759 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3760 | RT_BIT(X86_XCPT_BP)
3761 | RT_BIT(X86_XCPT_DB)
3762 | RT_BIT(X86_XCPT_DE)
3763 | RT_BIT(X86_XCPT_NM)
3764 | RT_BIT(X86_XCPT_TS)
3765 | RT_BIT(X86_XCPT_UD)
3766 | RT_BIT(X86_XCPT_NP)
3767 | RT_BIT(X86_XCPT_SS)
3768 | RT_BIT(X86_XCPT_GP)
3769 | RT_BIT(X86_XCPT_PF)
3770 | RT_BIT(X86_XCPT_MF)
3771 ;
3772#elif defined(HMVMX_ALWAYS_TRAP_PF)
3773 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3774#endif
3775
3776 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3777
3778 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3779 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3780 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3781 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3782 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3783 else
3784 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3785
3786 u32GuestCR0 |= uSetCR0;
3787 u32GuestCR0 &= uZapCR0;
3788 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3789
3790 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3791 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3792 AssertRCReturn(rc, rc);
3793 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3794 AssertRCReturn(rc, rc);
3795 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3796
3797 /*
3798 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3799 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3800 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3801 */
3802 uint32_t u32CR0Mask = 0;
3803 u32CR0Mask = X86_CR0_PE
3804 | X86_CR0_NE
3805 | X86_CR0_WP
3806 | X86_CR0_PG
3807 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3808 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3809 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3810
3811 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3812 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3813 * and @bugref{6944}. */
3814#if 0
3815 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3816 u32CR0Mask &= ~X86_CR0_PE;
3817#endif
3818 if (pVM->hm.s.fNestedPaging)
3819 u32CR0Mask &= ~X86_CR0_WP;
3820
3821 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3822 if (fInterceptNM)
3823 {
3824 u32CR0Mask |= X86_CR0_TS
3825 | X86_CR0_MP;
3826 }
3827
3828 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3829 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3830 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3831 AssertRCReturn(rc, rc);
3832 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3833
3834 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3835 }
3836 return rc;
3837}
3838
3839
3840/**
3841 * Loads the guest control registers (CR3, CR4) into the guest-state area
3842 * in the VMCS.
3843 *
3844 * @returns VBox status code.
3845 * @param pVM Pointer to the VM.
3846 * @param pVCpu Pointer to the VMCPU.
3847 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3848 * out-of-sync. Make sure to update the required fields
3849 * before using them.
3850 *
3851 * @remarks No-long-jump zone!!!
3852 */
3853static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3854{
3855 int rc = VINF_SUCCESS;
3856 PVM pVM = pVCpu->CTX_SUFF(pVM);
3857
3858 /*
3859 * Guest CR2.
3860 * It's always loaded in the assembler code. Nothing to do here.
3861 */
3862
3863 /*
3864 * Guest CR3.
3865 */
3866 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3867 {
3868 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3869 if (pVM->hm.s.fNestedPaging)
3870 {
3871 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3872
3873 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3874 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3875 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3876 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3877
3878 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3879 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3880 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3881
3882 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3883 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3884 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3885 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3886
3887 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3888 AssertRCReturn(rc, rc);
3889 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3890
3891 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3892 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3893 {
3894 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3895 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3896 {
3897 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3898 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3899 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3900 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3901 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3902 }
3903
3904 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3905 have Unrestricted Execution to handle the guest when it's not using paging. */
3906 GCPhysGuestCR3 = pMixedCtx->cr3;
3907 }
3908 else
3909 {
3910 /*
3911 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3912 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3913 * EPT takes care of translating it to host-physical addresses.
3914 */
3915 RTGCPHYS GCPhys;
3916 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3917 Assert(PDMVmmDevHeapIsEnabled(pVM));
3918
3919 /* We obtain it here every time as the guest could have relocated this PCI region. */
3920 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3921 AssertRCReturn(rc, rc);
3922
3923 GCPhysGuestCR3 = GCPhys;
3924 }
3925
3926 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3927 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3928 }
3929 else
3930 {
3931 /* Non-nested paging case, just use the hypervisor's CR3. */
3932 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3933
3934 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3935 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3936 }
3937 AssertRCReturn(rc, rc);
3938
3939 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3940 }
3941
3942 /*
3943 * Guest CR4.
3944 */
3945 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3946 {
3947 Assert(!(pMixedCtx->cr4 >> 32));
3948 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3949
3950 /* The guest's view of its CR4 is unblemished. */
3951 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3952 AssertRCReturn(rc, rc);
3953 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3954
3955 /* Setup VT-x's view of the guest CR4. */
3956 /*
3957 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3958 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3959 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3960 */
3961 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3962 {
3963 Assert(pVM->hm.s.vmx.pRealModeTSS);
3964 Assert(PDMVmmDevHeapIsEnabled(pVM));
3965 u32GuestCR4 &= ~X86_CR4_VME;
3966 }
3967
3968 if (pVM->hm.s.fNestedPaging)
3969 {
3970 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3971 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3972 {
3973 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3974 u32GuestCR4 |= X86_CR4_PSE;
3975 /* Our identity mapping is a 32-bit page directory. */
3976 u32GuestCR4 &= ~X86_CR4_PAE;
3977 }
3978 /* else use guest CR4.*/
3979 }
3980 else
3981 {
3982 /*
3983 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3984 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3985 */
3986 switch (pVCpu->hm.s.enmShadowMode)
3987 {
3988 case PGMMODE_REAL: /* Real-mode. */
3989 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3990 case PGMMODE_32_BIT: /* 32-bit paging. */
3991 {
3992 u32GuestCR4 &= ~X86_CR4_PAE;
3993 break;
3994 }
3995
3996 case PGMMODE_PAE: /* PAE paging. */
3997 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3998 {
3999 u32GuestCR4 |= X86_CR4_PAE;
4000 break;
4001 }
4002
4003 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4004 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4005#ifdef VBOX_ENABLE_64_BITS_GUESTS
4006 break;
4007#endif
4008 default:
4009 AssertFailed();
4010 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4011 }
4012 }
4013
4014 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4015 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4016 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4017 u32GuestCR4 |= uSetCR4;
4018 u32GuestCR4 &= uZapCR4;
4019
4020 /* Write VT-x's view of the guest CR4 into the VMCS. */
4021 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
4022 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4023 AssertRCReturn(rc, rc);
4024
4025 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4026 uint32_t u32CR4Mask = 0;
4027 u32CR4Mask = X86_CR4_VME
4028 | X86_CR4_PAE
4029 | X86_CR4_PGE
4030 | X86_CR4_PSE
4031 | X86_CR4_VMXE;
4032 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4033 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4034 AssertRCReturn(rc, rc);
4035
4036 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4037 }
4038 return rc;
4039}
4040
4041
4042/**
4043 * Loads the guest debug registers into the guest-state area in the VMCS.
4044 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4045 *
4046 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4047 *
4048 * @returns VBox status code.
4049 * @param pVCpu Pointer to the VMCPU.
4050 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4051 * out-of-sync. Make sure to update the required fields
4052 * before using them.
4053 *
4054 * @remarks No-long-jump zone!!!
4055 */
4056static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4057{
4058 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4059 return VINF_SUCCESS;
4060
4061#ifdef VBOX_STRICT
4062 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4063 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4064 {
4065 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4066 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4067 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4068 }
4069#endif
4070
4071 int rc;
4072 PVM pVM = pVCpu->CTX_SUFF(pVM);
4073 bool fInterceptDB = false;
4074 bool fInterceptMovDRx = false;
4075 if ( pVCpu->hm.s.fSingleInstruction
4076 || DBGFIsStepping(pVCpu))
4077 {
4078 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4079 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4080 {
4081 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4082 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4083 AssertRCReturn(rc, rc);
4084 Assert(fInterceptDB == false);
4085 }
4086 else
4087 {
4088 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4089 pVCpu->hm.s.fClearTrapFlag = true;
4090 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4091 fInterceptDB = true;
4092 }
4093 }
4094
4095 if ( fInterceptDB
4096 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4097 {
4098 /*
4099 * Use the combined guest and host DRx values found in the hypervisor
4100 * register set because the debugger has breakpoints active or someone
4101 * is single stepping on the host side without a monitor trap flag.
4102 *
4103 * Note! DBGF expects a clean DR6 state before executing guest code.
4104 */
4105#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4106 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4107 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4108 {
4109 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4110 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4111 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4112 }
4113 else
4114#endif
4115 if (!CPUMIsHyperDebugStateActive(pVCpu))
4116 {
4117 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4118 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4119 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4120 }
4121
4122 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4123 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4124 AssertRCReturn(rc, rc);
4125
4126 pVCpu->hm.s.fUsingHyperDR7 = true;
4127 fInterceptDB = true;
4128 fInterceptMovDRx = true;
4129 }
4130 else
4131 {
4132 /*
4133 * If the guest has enabled debug registers, we need to load them prior to
4134 * executing guest code so they'll trigger at the right time.
4135 */
4136 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4137 {
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4139 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4140 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146 }
4147 else
4148#endif
4149 if (!CPUMIsGuestDebugStateActive(pVCpu))
4150 {
4151 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155 }
4156 Assert(!fInterceptDB);
4157 Assert(!fInterceptMovDRx);
4158 }
4159 /*
4160 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4161 * must intercept #DB in order to maintain a correct DR6 guest value.
4162 */
4163#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4164 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4165 && !CPUMIsGuestDebugStateActive(pVCpu))
4166#else
4167 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4168#endif
4169 {
4170 fInterceptMovDRx = true;
4171 fInterceptDB = true;
4172 }
4173
4174 /* Update guest DR7. */
4175 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4176 AssertRCReturn(rc, rc);
4177
4178 pVCpu->hm.s.fUsingHyperDR7 = false;
4179 }
4180
4181 /*
4182 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4183 */
4184 if (fInterceptDB)
4185 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4186 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4187 {
4188#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4189 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4190#endif
4191 }
4192 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4193 AssertRCReturn(rc, rc);
4194
4195 /*
4196 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4197 */
4198 if (fInterceptMovDRx)
4199 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4200 else
4201 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4202 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4203 AssertRCReturn(rc, rc);
4204
4205 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4206 return VINF_SUCCESS;
4207}
4208
4209
4210#ifdef VBOX_STRICT
4211/**
4212 * Strict function to validate segment registers.
4213 *
4214 * @remarks ASSUMES CR0 is up to date.
4215 */
4216static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4217{
4218 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4219 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4220 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4221 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4222 && ( !CPUMIsGuestInRealModeEx(pCtx)
4223 && !CPUMIsGuestInV86ModeEx(pCtx)))
4224 {
4225 /* Protected mode checks */
4226 /* CS */
4227 Assert(pCtx->cs.Attr.n.u1Present);
4228 Assert(!(pCtx->cs.Attr.u & 0xf00));
4229 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4230 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4231 || !(pCtx->cs.Attr.n.u1Granularity));
4232 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4233 || (pCtx->cs.Attr.n.u1Granularity));
4234 /* CS cannot be loaded with NULL in protected mode. */
4235 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4236 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4237 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4238 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4239 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4240 else
4241 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4242 /* SS */
4243 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4244 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4245 if ( !(pCtx->cr0 & X86_CR0_PE)
4246 || pCtx->cs.Attr.n.u4Type == 3)
4247 {
4248 Assert(!pCtx->ss.Attr.n.u2Dpl);
4249 }
4250 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4251 {
4252 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4253 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4254 Assert(pCtx->ss.Attr.n.u1Present);
4255 Assert(!(pCtx->ss.Attr.u & 0xf00));
4256 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4257 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4258 || !(pCtx->ss.Attr.n.u1Granularity));
4259 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4260 || (pCtx->ss.Attr.n.u1Granularity));
4261 }
4262 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4263 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4264 {
4265 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4266 Assert(pCtx->ds.Attr.n.u1Present);
4267 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4268 Assert(!(pCtx->ds.Attr.u & 0xf00));
4269 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4270 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4271 || !(pCtx->ds.Attr.n.u1Granularity));
4272 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4273 || (pCtx->ds.Attr.n.u1Granularity));
4274 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4275 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4276 }
4277 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4278 {
4279 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4280 Assert(pCtx->es.Attr.n.u1Present);
4281 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4282 Assert(!(pCtx->es.Attr.u & 0xf00));
4283 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4284 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4285 || !(pCtx->es.Attr.n.u1Granularity));
4286 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4287 || (pCtx->es.Attr.n.u1Granularity));
4288 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4289 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4290 }
4291 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4292 {
4293 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4294 Assert(pCtx->fs.Attr.n.u1Present);
4295 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4296 Assert(!(pCtx->fs.Attr.u & 0xf00));
4297 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4298 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4299 || !(pCtx->fs.Attr.n.u1Granularity));
4300 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4301 || (pCtx->fs.Attr.n.u1Granularity));
4302 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4303 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4304 }
4305 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4306 {
4307 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4308 Assert(pCtx->gs.Attr.n.u1Present);
4309 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4310 Assert(!(pCtx->gs.Attr.u & 0xf00));
4311 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4312 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4313 || !(pCtx->gs.Attr.n.u1Granularity));
4314 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4315 || (pCtx->gs.Attr.n.u1Granularity));
4316 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4317 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4318 }
4319 /* 64-bit capable CPUs. */
4320# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4321 if (HMVMX_IS_64BIT_HOST_MODE())
4322 {
4323 Assert(!(pCtx->cs.u64Base >> 32));
4324 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4325 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4326 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4327 }
4328# endif
4329 }
4330 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4331 || ( CPUMIsGuestInRealModeEx(pCtx)
4332 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4333 {
4334 /* Real and v86 mode checks. */
4335 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4336 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4337 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4338 {
4339 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4340 }
4341 else
4342 {
4343 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4344 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4345 }
4346
4347 /* CS */
4348 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4349 Assert(pCtx->cs.u32Limit == 0xffff);
4350 Assert(u32CSAttr == 0xf3);
4351 /* SS */
4352 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4353 Assert(pCtx->ss.u32Limit == 0xffff);
4354 Assert(u32SSAttr == 0xf3);
4355 /* DS */
4356 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4357 Assert(pCtx->ds.u32Limit == 0xffff);
4358 Assert(u32DSAttr == 0xf3);
4359 /* ES */
4360 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4361 Assert(pCtx->es.u32Limit == 0xffff);
4362 Assert(u32ESAttr == 0xf3);
4363 /* FS */
4364 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4365 Assert(pCtx->fs.u32Limit == 0xffff);
4366 Assert(u32FSAttr == 0xf3);
4367 /* GS */
4368 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4369 Assert(pCtx->gs.u32Limit == 0xffff);
4370 Assert(u32GSAttr == 0xf3);
4371 /* 64-bit capable CPUs. */
4372# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4373 if (HMVMX_IS_64BIT_HOST_MODE())
4374 {
4375 Assert(!(pCtx->cs.u64Base >> 32));
4376 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4377 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4378 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4379 }
4380# endif
4381 }
4382}
4383#endif /* VBOX_STRICT */
4384
4385
4386/**
4387 * Writes a guest segment register into the guest-state area in the VMCS.
4388 *
4389 * @returns VBox status code.
4390 * @param pVCpu Pointer to the VMCPU.
4391 * @param idxSel Index of the selector in the VMCS.
4392 * @param idxLimit Index of the segment limit in the VMCS.
4393 * @param idxBase Index of the segment base in the VMCS.
4394 * @param idxAccess Index of the access rights of the segment in the VMCS.
4395 * @param pSelReg Pointer to the segment selector.
4396 *
4397 * @remarks No-long-jump zone!!!
4398 */
4399static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4400 uint32_t idxAccess, PCPUMSELREG pSelReg)
4401{
4402 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4403 AssertRCReturn(rc, rc);
4404 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4405 AssertRCReturn(rc, rc);
4406 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4407 AssertRCReturn(rc, rc);
4408
4409 uint32_t u32Access = pSelReg->Attr.u;
4410 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4411 {
4412 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4413 u32Access = 0xf3;
4414 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4415 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4416 }
4417 else
4418 {
4419 /*
4420 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4421 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4422 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4423 * loaded in protected-mode have their attribute as 0.
4424 */
4425 if (!u32Access)
4426 u32Access = X86DESCATTR_UNUSABLE;
4427 }
4428
4429 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4430 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4431 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4432
4433 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4434 AssertRCReturn(rc, rc);
4435 return rc;
4436}
4437
4438
4439/**
4440 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4441 * into the guest-state area in the VMCS.
4442 *
4443 * @returns VBox status code.
4444 * @param pVM Pointer to the VM.
4445 * @param pVCPU Pointer to the VMCPU.
4446 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4447 * out-of-sync. Make sure to update the required fields
4448 * before using them.
4449 *
4450 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4451 * @remarks No-long-jump zone!!!
4452 */
4453static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4454{
4455 int rc = VERR_INTERNAL_ERROR_5;
4456 PVM pVM = pVCpu->CTX_SUFF(pVM);
4457
4458 /*
4459 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4460 */
4461 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4462 {
4463 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4464 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4465 {
4466 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4467 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4468 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4469 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4470 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4471 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4472 }
4473
4474#ifdef VBOX_WITH_REM
4475 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4476 {
4477 Assert(pVM->hm.s.vmx.pRealModeTSS);
4478 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4479 if ( pVCpu->hm.s.vmx.fWasInRealMode
4480 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4481 {
4482 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4483 in real-mode (e.g. OpenBSD 4.0) */
4484 REMFlushTBs(pVM);
4485 Log4(("Load: Switch to protected mode detected!\n"));
4486 pVCpu->hm.s.vmx.fWasInRealMode = false;
4487 }
4488 }
4489#endif
4490 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4491 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4492 AssertRCReturn(rc, rc);
4493 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4494 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4495 AssertRCReturn(rc, rc);
4496 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4497 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4498 AssertRCReturn(rc, rc);
4499 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4500 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4501 AssertRCReturn(rc, rc);
4502 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4503 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4504 AssertRCReturn(rc, rc);
4505 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4506 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4507 AssertRCReturn(rc, rc);
4508
4509#ifdef VBOX_STRICT
4510 /* Validate. */
4511 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4512#endif
4513
4514 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4515 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4516 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4517 }
4518
4519 /*
4520 * Guest TR.
4521 */
4522 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4523 {
4524 /*
4525 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4526 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4527 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4528 */
4529 uint16_t u16Sel = 0;
4530 uint32_t u32Limit = 0;
4531 uint64_t u64Base = 0;
4532 uint32_t u32AccessRights = 0;
4533
4534 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4535 {
4536 u16Sel = pMixedCtx->tr.Sel;
4537 u32Limit = pMixedCtx->tr.u32Limit;
4538 u64Base = pMixedCtx->tr.u64Base;
4539 u32AccessRights = pMixedCtx->tr.Attr.u;
4540 }
4541 else
4542 {
4543 Assert(pVM->hm.s.vmx.pRealModeTSS);
4544 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4545
4546 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4547 RTGCPHYS GCPhys;
4548 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4549 AssertRCReturn(rc, rc);
4550
4551 X86DESCATTR DescAttr;
4552 DescAttr.u = 0;
4553 DescAttr.n.u1Present = 1;
4554 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4555
4556 u16Sel = 0;
4557 u32Limit = HM_VTX_TSS_SIZE;
4558 u64Base = GCPhys; /* in real-mode phys = virt. */
4559 u32AccessRights = DescAttr.u;
4560 }
4561
4562 /* Validate. */
4563 Assert(!(u16Sel & RT_BIT(2)));
4564 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4565 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4566 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4567 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4568 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4569 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4570 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4571 Assert( (u32Limit & 0xfff) == 0xfff
4572 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4573 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4574 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4575
4576 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4577 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4578 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4579 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4580
4581 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4582 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4583 }
4584
4585 /*
4586 * Guest GDTR.
4587 */
4588 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4589 {
4590 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4591 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4592
4593 /* Validate. */
4594 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4595
4596 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4597 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4598 }
4599
4600 /*
4601 * Guest LDTR.
4602 */
4603 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4604 {
4605 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4606 uint32_t u32Access = 0;
4607 if (!pMixedCtx->ldtr.Attr.u)
4608 u32Access = X86DESCATTR_UNUSABLE;
4609 else
4610 u32Access = pMixedCtx->ldtr.Attr.u;
4611
4612 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4615 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4616
4617 /* Validate. */
4618 if (!(u32Access & X86DESCATTR_UNUSABLE))
4619 {
4620 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4621 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4622 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4623 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4624 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4625 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4626 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4627 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4628 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4629 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4630 }
4631
4632 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4633 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4634 }
4635
4636 /*
4637 * Guest IDTR.
4638 */
4639 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4640 {
4641 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4642 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4643
4644 /* Validate. */
4645 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4646
4647 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4648 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4649 }
4650
4651 return VINF_SUCCESS;
4652}
4653
4654
4655/**
4656 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4657 * areas. These MSRs will automatically be loaded to the host CPU on every
4658 * successful VM entry and stored from the host CPU on every successful VM-exit.
4659 *
4660 * This also creates/updates MSR slots for the host MSRs. The actual host
4661 * MSR values are -not- updated here for performance reasons. See
4662 * hmR0VmxSaveHostMsrs().
4663 *
4664 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4665 *
4666 * @returns VBox status code.
4667 * @param pVCpu Pointer to the VMCPU.
4668 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4669 * out-of-sync. Make sure to update the required fields
4670 * before using them.
4671 *
4672 * @remarks No-long-jump zone!!!
4673 */
4674static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4675{
4676 AssertPtr(pVCpu);
4677 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4678
4679 /*
4680 * MSRs that we use the auto-load/store MSR area in the VMCS.
4681 */
4682 PVM pVM = pVCpu->CTX_SUFF(pVM);
4683 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4684 {
4685 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4686#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4687 if (pVM->hm.s.fAllow64BitGuests)
4688 {
4689 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4690 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4691 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4692 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4693# ifdef DEBUG
4694 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4695 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4696 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4697# endif
4698 }
4699#endif
4700 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4701 }
4702
4703 /*
4704 * Guest Sysenter MSRs.
4705 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4706 * VM-exits on WRMSRs for these MSRs.
4707 */
4708 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4709 {
4710 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4711 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4712 }
4713
4714 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4715 {
4716 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4717 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4718 }
4719
4720 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4721 {
4722 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4723 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4724 }
4725
4726 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4727 {
4728 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4729 {
4730 /*
4731 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4732 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4733 */
4734 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4735 {
4736 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4737 AssertRCReturn(rc,rc);
4738 Log4(("Load: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pMixedCtx->msrEFER));
4739 }
4740 else
4741 {
4742 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4743 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4744 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4745 Log4(("Load: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4746 pVCpu->hm.s.vmx.cMsrs));
4747 }
4748 }
4749 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4750 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4751 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4752 }
4753
4754 return VINF_SUCCESS;
4755}
4756
4757
4758/**
4759 * Loads the guest activity state into the guest-state area in the VMCS.
4760 *
4761 * @returns VBox status code.
4762 * @param pVCpu Pointer to the VMCPU.
4763 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4764 * out-of-sync. Make sure to update the required fields
4765 * before using them.
4766 *
4767 * @remarks No-long-jump zone!!!
4768 */
4769static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4770{
4771 NOREF(pCtx);
4772 /** @todo See if we can make use of other states, e.g.
4773 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4774 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4775 {
4776 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4777 AssertRCReturn(rc, rc);
4778
4779 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4780 }
4781 return VINF_SUCCESS;
4782}
4783
4784
4785/**
4786 * Sets up the appropriate function to run guest code.
4787 *
4788 * @returns VBox status code.
4789 * @param pVCpu Pointer to the VMCPU.
4790 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4791 * out-of-sync. Make sure to update the required fields
4792 * before using them.
4793 *
4794 * @remarks No-long-jump zone!!!
4795 */
4796static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4797{
4798 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4799 {
4800#ifndef VBOX_ENABLE_64_BITS_GUESTS
4801 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4802#endif
4803 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4804#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4805 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4806 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4807 {
4808 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4809 {
4810 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4811 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4812 | HM_CHANGED_VMX_EXIT_CTLS
4813 | HM_CHANGED_VMX_ENTRY_CTLS
4814 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4815 }
4816 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4817 }
4818#else
4819 /* 64-bit host or hybrid host. */
4820 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4821#endif
4822 }
4823 else
4824 {
4825 /* Guest is not in long mode, use the 32-bit handler. */
4826#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4827 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4828 {
4829 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4830 {
4831 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4832 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4833 | HM_CHANGED_VMX_EXIT_CTLS
4834 | HM_CHANGED_VMX_ENTRY_CTLS
4835 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4836 }
4837 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4838 }
4839#else
4840 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4841#endif
4842 }
4843 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4844 return VINF_SUCCESS;
4845}
4846
4847
4848/**
4849 * Wrapper for running the guest code in VT-x.
4850 *
4851 * @returns VBox strict status code.
4852 * @param pVM Pointer to the VM.
4853 * @param pVCpu Pointer to the VMCPU.
4854 * @param pCtx Pointer to the guest-CPU context.
4855 *
4856 * @remarks No-long-jump zone!!!
4857 */
4858DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4859{
4860 /*
4861 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4862 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4863 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4864 */
4865 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4866 /** @todo Add stats for resume vs launch. */
4867#ifdef VBOX_WITH_KERNEL_USING_XMM
4868 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4869#else
4870 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4871#endif
4872}
4873
4874
4875/**
4876 * Reports world-switch error and dumps some useful debug info.
4877 *
4878 * @param pVM Pointer to the VM.
4879 * @param pVCpu Pointer to the VMCPU.
4880 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4881 * @param pCtx Pointer to the guest-CPU context.
4882 * @param pVmxTransient Pointer to the VMX transient structure (only
4883 * exitReason updated).
4884 */
4885static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4886{
4887 Assert(pVM);
4888 Assert(pVCpu);
4889 Assert(pCtx);
4890 Assert(pVmxTransient);
4891 HMVMX_ASSERT_PREEMPT_SAFE();
4892
4893 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4894 switch (rcVMRun)
4895 {
4896 case VERR_VMX_INVALID_VMXON_PTR:
4897 AssertFailed();
4898 break;
4899 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4900 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4901 {
4902 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4903 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4904 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4905 AssertRC(rc);
4906
4907 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4908 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4909 Cannot do it here as we may have been long preempted. */
4910
4911#ifdef VBOX_STRICT
4912 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4913 pVmxTransient->uExitReason));
4914 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4915 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4916 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4917 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4918 else
4919 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4920 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4921 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4922
4923 /* VMX control bits. */
4924 uint32_t u32Val;
4925 uint64_t u64Val;
4926 HMVMXHCUINTREG uHCReg;
4927 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4928 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4929 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4930 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4931 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4932 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4933 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4934 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4935 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4936 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4937 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4938 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4939 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4940 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4941 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4942 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4943 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4944 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4945 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4946 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4947 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4948 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4949 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4950 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4951 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4952 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4953 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4954 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4955 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4956 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4957 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4958 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4959 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4960 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4961 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4962 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4963 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4964 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4965 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4966 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4967 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4968 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4969
4970 /* Guest bits. */
4971 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4972 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4973 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4974 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4975 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4976 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4977 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4978 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4979
4980 /* Host bits. */
4981 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4982 Log4(("Host CR0 %#RHr\n", uHCReg));
4983 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4984 Log4(("Host CR3 %#RHr\n", uHCReg));
4985 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4986 Log4(("Host CR4 %#RHr\n", uHCReg));
4987
4988 RTGDTR HostGdtr;
4989 PCX86DESCHC pDesc;
4990 ASMGetGDTR(&HostGdtr);
4991 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4992 Log4(("Host CS %#08x\n", u32Val));
4993 if (u32Val < HostGdtr.cbGdt)
4994 {
4995 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4996 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4997 }
4998
4999 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5000 Log4(("Host DS %#08x\n", u32Val));
5001 if (u32Val < HostGdtr.cbGdt)
5002 {
5003 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5004 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5005 }
5006
5007 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5008 Log4(("Host ES %#08x\n", u32Val));
5009 if (u32Val < HostGdtr.cbGdt)
5010 {
5011 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5012 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5013 }
5014
5015 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5016 Log4(("Host FS %#08x\n", u32Val));
5017 if (u32Val < HostGdtr.cbGdt)
5018 {
5019 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5020 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5021 }
5022
5023 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5024 Log4(("Host GS %#08x\n", u32Val));
5025 if (u32Val < HostGdtr.cbGdt)
5026 {
5027 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5028 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5029 }
5030
5031 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5032 Log4(("Host SS %#08x\n", u32Val));
5033 if (u32Val < HostGdtr.cbGdt)
5034 {
5035 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5036 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5037 }
5038
5039 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5040 Log4(("Host TR %#08x\n", u32Val));
5041 if (u32Val < HostGdtr.cbGdt)
5042 {
5043 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5044 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5045 }
5046
5047 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5048 Log4(("Host TR Base %#RHv\n", uHCReg));
5049 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5050 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5051 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5052 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5053 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5054 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5056 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5057 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5058 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5060 Log4(("Host RSP %#RHv\n", uHCReg));
5061 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5062 Log4(("Host RIP %#RHv\n", uHCReg));
5063# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5064 if (HMVMX_IS_64BIT_HOST_MODE())
5065 {
5066 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5067 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5068 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5069 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5070 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5071 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5072 }
5073# endif
5074#endif /* VBOX_STRICT */
5075 break;
5076 }
5077
5078 default:
5079 /* Impossible */
5080 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5081 break;
5082 }
5083 NOREF(pVM); NOREF(pCtx);
5084}
5085
5086
5087#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5088#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5089# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5090#endif
5091#ifdef VBOX_STRICT
5092static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5093{
5094 switch (idxField)
5095 {
5096 case VMX_VMCS_GUEST_RIP:
5097 case VMX_VMCS_GUEST_RSP:
5098 case VMX_VMCS_GUEST_SYSENTER_EIP:
5099 case VMX_VMCS_GUEST_SYSENTER_ESP:
5100 case VMX_VMCS_GUEST_GDTR_BASE:
5101 case VMX_VMCS_GUEST_IDTR_BASE:
5102 case VMX_VMCS_GUEST_CS_BASE:
5103 case VMX_VMCS_GUEST_DS_BASE:
5104 case VMX_VMCS_GUEST_ES_BASE:
5105 case VMX_VMCS_GUEST_FS_BASE:
5106 case VMX_VMCS_GUEST_GS_BASE:
5107 case VMX_VMCS_GUEST_SS_BASE:
5108 case VMX_VMCS_GUEST_LDTR_BASE:
5109 case VMX_VMCS_GUEST_TR_BASE:
5110 case VMX_VMCS_GUEST_CR3:
5111 return true;
5112 }
5113 return false;
5114}
5115
5116static bool hmR0VmxIsValidReadField(uint32_t idxField)
5117{
5118 switch (idxField)
5119 {
5120 /* Read-only fields. */
5121 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5122 return true;
5123 }
5124 /* Remaining readable fields should also be writable. */
5125 return hmR0VmxIsValidWriteField(idxField);
5126}
5127#endif /* VBOX_STRICT */
5128
5129
5130/**
5131 * Executes the specified handler in 64-bit mode.
5132 *
5133 * @returns VBox status code.
5134 * @param pVM Pointer to the VM.
5135 * @param pVCpu Pointer to the VMCPU.
5136 * @param pCtx Pointer to the guest CPU context.
5137 * @param enmOp The operation to perform.
5138 * @param cbParam Number of parameters.
5139 * @param paParam Array of 32-bit parameters.
5140 */
5141VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5142 uint32_t *paParam)
5143{
5144 int rc, rc2;
5145 PHMGLOBALCPUINFO pCpu;
5146 RTHCPHYS HCPhysCpuPage;
5147 RTCCUINTREG uOldEflags;
5148
5149 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5150 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5151 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5152 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5153
5154#ifdef VBOX_STRICT
5155 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5156 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5157
5158 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5159 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5160#endif
5161
5162 /* Disable interrupts. */
5163 uOldEflags = ASMIntDisableFlags();
5164
5165#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5166 RTCPUID idHostCpu = RTMpCpuId();
5167 CPUMR0SetLApic(pVCpu, idHostCpu);
5168#endif
5169
5170 pCpu = HMR0GetCurrentCpu();
5171 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5172
5173 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5174 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5175
5176 /* Leave VMX Root Mode. */
5177 VMXDisable();
5178
5179 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5180
5181 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5182 CPUMSetHyperEIP(pVCpu, enmOp);
5183 for (int i = (int)cbParam - 1; i >= 0; i--)
5184 CPUMPushHyper(pVCpu, paParam[i]);
5185
5186 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5187
5188 /* Call the switcher. */
5189 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5190 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5191
5192 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5193 /* Make sure the VMX instructions don't cause #UD faults. */
5194 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5195
5196 /* Re-enter VMX Root Mode */
5197 rc2 = VMXEnable(HCPhysCpuPage);
5198 if (RT_FAILURE(rc2))
5199 {
5200 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5201 ASMSetFlags(uOldEflags);
5202 return rc2;
5203 }
5204
5205 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5206 AssertRC(rc2);
5207 Assert(!(ASMGetFlags() & X86_EFL_IF));
5208 ASMSetFlags(uOldEflags);
5209 return rc;
5210}
5211
5212
5213/**
5214 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5215 * supporting 64-bit guests.
5216 *
5217 * @returns VBox status code.
5218 * @param fResume Whether to VMLAUNCH or VMRESUME.
5219 * @param pCtx Pointer to the guest-CPU context.
5220 * @param pCache Pointer to the VMCS cache.
5221 * @param pVM Pointer to the VM.
5222 * @param pVCpu Pointer to the VMCPU.
5223 */
5224DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5225{
5226 uint32_t aParam[6];
5227 PHMGLOBALCPUINFO pCpu = NULL;
5228 RTHCPHYS HCPhysCpuPage = 0;
5229 int rc = VERR_INTERNAL_ERROR_5;
5230
5231 pCpu = HMR0GetCurrentCpu();
5232 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5233
5234#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5235 pCache->uPos = 1;
5236 pCache->interPD = PGMGetInterPaeCR3(pVM);
5237 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5238#endif
5239
5240#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5241 pCache->TestIn.HCPhysCpuPage = 0;
5242 pCache->TestIn.HCPhysVmcs = 0;
5243 pCache->TestIn.pCache = 0;
5244 pCache->TestOut.HCPhysVmcs = 0;
5245 pCache->TestOut.pCache = 0;
5246 pCache->TestOut.pCtx = 0;
5247 pCache->TestOut.eflags = 0;
5248#endif
5249
5250 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5251 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5252 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5253 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5254 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5255 aParam[5] = 0;
5256
5257#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5258 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5259 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5260#endif
5261 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5262
5263#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5264 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5265 Assert(pCtx->dr[4] == 10);
5266 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5267#endif
5268
5269#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5270 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5271 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5272 pVCpu->hm.s.vmx.HCPhysVmcs));
5273 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5274 pCache->TestOut.HCPhysVmcs));
5275 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5276 pCache->TestOut.pCache));
5277 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5278 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5279 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5280 pCache->TestOut.pCtx));
5281 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5282#endif
5283 return rc;
5284}
5285
5286
5287/**
5288 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5289 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5290 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5291 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5292 *
5293 * @returns VBox status code.
5294 * @param pVM Pointer to the VM.
5295 * @param pVCpu Pointer to the VMCPU.
5296 */
5297static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5298{
5299#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5300{ \
5301 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5302 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5303 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5304 ++cReadFields; \
5305}
5306
5307 AssertPtr(pVM);
5308 AssertPtr(pVCpu);
5309 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5310 uint32_t cReadFields = 0;
5311
5312 /*
5313 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5314 * and serve to indicate exceptions to the rules.
5315 */
5316
5317 /* Guest-natural selector base fields. */
5318#if 0
5319 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5320 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5321 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5322#endif
5323 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5324 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5325 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5326 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5327 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5328 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5329 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5330 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5331 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5332 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5333 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5334 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5335#if 0
5336 /* Unused natural width guest-state fields. */
5337 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5338 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5339#endif
5340 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5341 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5342
5343 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5344#if 0
5345 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5346 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5347 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5348 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5349 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5350 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5351 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5352 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5353 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5354#endif
5355
5356 /* Natural width guest-state fields. */
5357 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5358#if 0
5359 /* Currently unused field. */
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5361#endif
5362
5363 if (pVM->hm.s.fNestedPaging)
5364 {
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5366 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5367 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5368 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5369 }
5370 else
5371 {
5372 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5373 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5374 }
5375
5376#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5377 return VINF_SUCCESS;
5378}
5379
5380
5381/**
5382 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5383 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5384 * darwin, running 64-bit guests).
5385 *
5386 * @returns VBox status code.
5387 * @param pVCpu Pointer to the VMCPU.
5388 * @param idxField The VMCS field encoding.
5389 * @param u64Val 16, 32 or 64-bit value.
5390 */
5391VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5392{
5393 int rc;
5394 switch (idxField)
5395 {
5396 /*
5397 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5398 */
5399 /* 64-bit Control fields. */
5400 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5401 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5402 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5403 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5404 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5405 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5406 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5407 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5408 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5409 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5410 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5411 case VMX_VMCS64_CTRL_EPTP_FULL:
5412 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5413 /* 64-bit Guest-state fields. */
5414 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5415 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5416 case VMX_VMCS64_GUEST_PAT_FULL:
5417 case VMX_VMCS64_GUEST_EFER_FULL:
5418 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5419 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5420 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5421 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5422 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5423 /* 64-bit Host-state fields. */
5424 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5425 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5426 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5427 {
5428 rc = VMXWriteVmcs32(idxField, u64Val);
5429 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5430 break;
5431 }
5432
5433 /*
5434 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5435 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5436 */
5437 /* Natural-width Guest-state fields. */
5438 case VMX_VMCS_GUEST_CR3:
5439 case VMX_VMCS_GUEST_ES_BASE:
5440 case VMX_VMCS_GUEST_CS_BASE:
5441 case VMX_VMCS_GUEST_SS_BASE:
5442 case VMX_VMCS_GUEST_DS_BASE:
5443 case VMX_VMCS_GUEST_FS_BASE:
5444 case VMX_VMCS_GUEST_GS_BASE:
5445 case VMX_VMCS_GUEST_LDTR_BASE:
5446 case VMX_VMCS_GUEST_TR_BASE:
5447 case VMX_VMCS_GUEST_GDTR_BASE:
5448 case VMX_VMCS_GUEST_IDTR_BASE:
5449 case VMX_VMCS_GUEST_RSP:
5450 case VMX_VMCS_GUEST_RIP:
5451 case VMX_VMCS_GUEST_SYSENTER_ESP:
5452 case VMX_VMCS_GUEST_SYSENTER_EIP:
5453 {
5454 if (!(u64Val >> 32))
5455 {
5456 /* If this field is 64-bit, VT-x will zero out the top bits. */
5457 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5458 }
5459 else
5460 {
5461 /* Assert that only the 32->64 switcher case should ever come here. */
5462 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5463 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5464 }
5465 break;
5466 }
5467
5468 default:
5469 {
5470 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5471 rc = VERR_INVALID_PARAMETER;
5472 break;
5473 }
5474 }
5475 AssertRCReturn(rc, rc);
5476 return rc;
5477}
5478
5479
5480/**
5481 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5482 * hosts (except darwin) for 64-bit guests.
5483 *
5484 * @param pVCpu Pointer to the VMCPU.
5485 * @param idxField The VMCS field encoding.
5486 * @param u64Val 16, 32 or 64-bit value.
5487 */
5488VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5489{
5490 AssertPtr(pVCpu);
5491 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5492
5493 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5494 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5495
5496 /* Make sure there are no duplicates. */
5497 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5498 {
5499 if (pCache->Write.aField[i] == idxField)
5500 {
5501 pCache->Write.aFieldVal[i] = u64Val;
5502 return VINF_SUCCESS;
5503 }
5504 }
5505
5506 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5507 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5508 pCache->Write.cValidEntries++;
5509 return VINF_SUCCESS;
5510}
5511
5512/* Enable later when the assembly code uses these as callbacks. */
5513#if 0
5514/*
5515 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5516 *
5517 * @param pVCpu Pointer to the VMCPU.
5518 * @param pCache Pointer to the VMCS cache.
5519 *
5520 * @remarks No-long-jump zone!!!
5521 */
5522VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5523{
5524 AssertPtr(pCache);
5525 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5526 {
5527 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5528 AssertRC(rc);
5529 }
5530 pCache->Write.cValidEntries = 0;
5531}
5532
5533
5534/**
5535 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5536 *
5537 * @param pVCpu Pointer to the VMCPU.
5538 * @param pCache Pointer to the VMCS cache.
5539 *
5540 * @remarks No-long-jump zone!!!
5541 */
5542VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5543{
5544 AssertPtr(pCache);
5545 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5546 {
5547 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5548 AssertRC(rc);
5549 }
5550}
5551#endif
5552#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5553
5554
5555/**
5556 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5557 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5558 * timer.
5559 *
5560 * @returns VBox status code.
5561 * @param pVCpu Pointer to the VMCPU.
5562 *
5563 * @remarks No-long-jump zone!!!
5564 */
5565static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5566{
5567 int rc = VERR_INTERNAL_ERROR_5;
5568 bool fOffsettedTsc = false;
5569 bool fParavirtTsc = false;
5570 PVM pVM = pVCpu->CTX_SUFF(pVM);
5571 if (pVM->hm.s.vmx.fUsePreemptTimer)
5572 {
5573 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5574 &pVCpu->hm.s.vmx.u64TSCOffset);
5575
5576 /* Make sure the returned values have sane upper and lower boundaries. */
5577 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5578 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5579 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5580 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5581
5582 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5583 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5584 }
5585 else
5586 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5587
5588#if 0
5589 if (fParavirtTsc)
5590 {
5591 uint64_t const u64CurTsc = ASMReadTSC();
5592 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5593 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5594 {
5595 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5596 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5597 }
5598
5599 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5600 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5601 AssertRC(rc);
5602 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5603 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5604
5605 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5606 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5607 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5608 }
5609 else
5610#endif
5611
5612 if (fParavirtTsc)
5613 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5614 if (fOffsettedTsc)
5615 {
5616 uint64_t u64CurTSC = ASMReadTSC();
5617 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5618 {
5619 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5620 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5621
5622 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5623 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5624 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5625 }
5626 else
5627 {
5628 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5629 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5630 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5631 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5632 }
5633 }
5634 else
5635 {
5636 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5637 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5639 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5640 }
5641}
5642
5643
5644/**
5645 * Determines if an exception is a contributory exception. Contributory
5646 * exceptions are ones which can cause double-faults. Page-fault is
5647 * intentionally not included here as it's a conditional contributory exception.
5648 *
5649 * @returns true if the exception is contributory, false otherwise.
5650 * @param uVector The exception vector.
5651 */
5652DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5653{
5654 switch (uVector)
5655 {
5656 case X86_XCPT_GP:
5657 case X86_XCPT_SS:
5658 case X86_XCPT_NP:
5659 case X86_XCPT_TS:
5660 case X86_XCPT_DE:
5661 return true;
5662 default:
5663 break;
5664 }
5665 return false;
5666}
5667
5668
5669/**
5670 * Sets an event as a pending event to be injected into the guest.
5671 *
5672 * @param pVCpu Pointer to the VMCPU.
5673 * @param u32IntInfo The VM-entry interruption-information field.
5674 * @param cbInstr The VM-entry instruction length in bytes (for software
5675 * interrupts, exceptions and privileged software
5676 * exceptions).
5677 * @param u32ErrCode The VM-entry exception error code.
5678 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5679 * page-fault.
5680 *
5681 * @remarks Statistics counter assumes this is a guest event being injected or
5682 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5683 * always incremented.
5684 */
5685DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5686 RTGCUINTPTR GCPtrFaultAddress)
5687{
5688 Assert(!pVCpu->hm.s.Event.fPending);
5689 pVCpu->hm.s.Event.fPending = true;
5690 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5691 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5692 pVCpu->hm.s.Event.cbInstr = cbInstr;
5693 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5694
5695 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5696}
5697
5698
5699/**
5700 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5701 *
5702 * @param pVCpu Pointer to the VMCPU.
5703 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5704 * out-of-sync. Make sure to update the required fields
5705 * before using them.
5706 */
5707DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5708{
5709 NOREF(pMixedCtx);
5710 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5711 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5712 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5713 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5714}
5715
5716
5717/**
5718 * Handle a condition that occurred while delivering an event through the guest
5719 * IDT.
5720 *
5721 * @returns VBox status code (informational error codes included).
5722 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5723 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5724 * continue execution of the guest which will delivery the #DF.
5725 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5726 *
5727 * @param pVCpu Pointer to the VMCPU.
5728 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5729 * out-of-sync. Make sure to update the required fields
5730 * before using them.
5731 * @param pVmxTransient Pointer to the VMX transient structure.
5732 *
5733 * @remarks No-long-jump zone!!!
5734 */
5735static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5736{
5737 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5738 AssertRCReturn(rc, rc);
5739 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5740 {
5741 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5742 AssertRCReturn(rc, rc);
5743
5744 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5745 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5746 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5747
5748 typedef enum
5749 {
5750 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5751 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5752 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5753 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5754 } VMXREFLECTXCPT;
5755
5756 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5757 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5758 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5759 {
5760 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5761 {
5762 enmReflect = VMXREFLECTXCPT_XCPT;
5763#ifdef VBOX_STRICT
5764 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5765 && uExitVector == X86_XCPT_PF)
5766 {
5767 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5768 }
5769#endif
5770 if ( uExitVector == X86_XCPT_PF
5771 && uIdtVector == X86_XCPT_PF)
5772 {
5773 pVmxTransient->fVectoringPF = true;
5774 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5775 }
5776 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5777 && hmR0VmxIsContributoryXcpt(uExitVector)
5778 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5779 || uIdtVector == X86_XCPT_PF))
5780 {
5781 enmReflect = VMXREFLECTXCPT_DF;
5782 }
5783 else if (uIdtVector == X86_XCPT_DF)
5784 enmReflect = VMXREFLECTXCPT_TF;
5785 }
5786 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5787 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5788 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5789 {
5790 /*
5791 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5792 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5793 */
5794 enmReflect = VMXREFLECTXCPT_XCPT;
5795 }
5796 }
5797 else
5798 {
5799 /*
5800 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5801 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5802 * original exception to the guest after handling the VM-exit.
5803 */
5804 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5805 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5806 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5807 {
5808 enmReflect = VMXREFLECTXCPT_XCPT;
5809 }
5810 }
5811
5812 switch (enmReflect)
5813 {
5814 case VMXREFLECTXCPT_XCPT:
5815 {
5816 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5817 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5818 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5819
5820 uint32_t u32ErrCode = 0;
5821 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5822 {
5823 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5824 AssertRCReturn(rc, rc);
5825 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5826 }
5827
5828 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5829 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5830 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5831 rc = VINF_SUCCESS;
5832 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5833 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5834
5835 break;
5836 }
5837
5838 case VMXREFLECTXCPT_DF:
5839 {
5840 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5841 rc = VINF_HM_DOUBLE_FAULT;
5842 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5843 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5844
5845 break;
5846 }
5847
5848 case VMXREFLECTXCPT_TF:
5849 {
5850 rc = VINF_EM_RESET;
5851 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5852 uExitVector));
5853 break;
5854 }
5855
5856 default:
5857 Assert(rc == VINF_SUCCESS);
5858 break;
5859 }
5860 }
5861 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5862 return rc;
5863}
5864
5865
5866/**
5867 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5868 *
5869 * @returns VBox status code.
5870 * @param pVCpu Pointer to the VMCPU.
5871 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5872 * out-of-sync. Make sure to update the required fields
5873 * before using them.
5874 *
5875 * @remarks No-long-jump zone!!!
5876 */
5877static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5878{
5879 NOREF(pMixedCtx);
5880
5881 /*
5882 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5883 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5884 */
5885 VMMRZCallRing3Disable(pVCpu);
5886 HM_DISABLE_PREEMPT_IF_NEEDED();
5887
5888 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5889 {
5890 uint32_t uVal = 0;
5891 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5892 AssertRCReturn(rc, rc);
5893
5894 uint32_t uShadow = 0;
5895 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5896 AssertRCReturn(rc, rc);
5897
5898 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5899 CPUMSetGuestCR0(pVCpu, uVal);
5900 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5901 }
5902
5903 HM_RESTORE_PREEMPT_IF_NEEDED();
5904 VMMRZCallRing3Enable(pVCpu);
5905 return VINF_SUCCESS;
5906}
5907
5908
5909/**
5910 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5911 *
5912 * @returns VBox status code.
5913 * @param pVCpu Pointer to the VMCPU.
5914 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5915 * out-of-sync. Make sure to update the required fields
5916 * before using them.
5917 *
5918 * @remarks No-long-jump zone!!!
5919 */
5920static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5921{
5922 NOREF(pMixedCtx);
5923
5924 int rc = VINF_SUCCESS;
5925 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5926 {
5927 uint32_t uVal = 0;
5928 uint32_t uShadow = 0;
5929 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5930 AssertRCReturn(rc, rc);
5931 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5932 AssertRCReturn(rc, rc);
5933
5934 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5935 CPUMSetGuestCR4(pVCpu, uVal);
5936 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5937 }
5938 return rc;
5939}
5940
5941
5942/**
5943 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5944 *
5945 * @returns VBox status code.
5946 * @param pVCpu Pointer to the VMCPU.
5947 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5948 * out-of-sync. Make sure to update the required fields
5949 * before using them.
5950 *
5951 * @remarks No-long-jump zone!!!
5952 */
5953static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5954{
5955 int rc = VINF_SUCCESS;
5956 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5957 {
5958 uint64_t u64Val = 0;
5959 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5960 AssertRCReturn(rc, rc);
5961
5962 pMixedCtx->rip = u64Val;
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5964 }
5965 return rc;
5966}
5967
5968
5969/**
5970 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5971 *
5972 * @returns VBox status code.
5973 * @param pVCpu Pointer to the VMCPU.
5974 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5975 * out-of-sync. Make sure to update the required fields
5976 * before using them.
5977 *
5978 * @remarks No-long-jump zone!!!
5979 */
5980static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5981{
5982 int rc = VINF_SUCCESS;
5983 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5984 {
5985 uint64_t u64Val = 0;
5986 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5987 AssertRCReturn(rc, rc);
5988
5989 pMixedCtx->rsp = u64Val;
5990 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5991 }
5992 return rc;
5993}
5994
5995
5996/**
5997 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5998 *
5999 * @returns VBox status code.
6000 * @param pVCpu Pointer to the VMCPU.
6001 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6002 * out-of-sync. Make sure to update the required fields
6003 * before using them.
6004 *
6005 * @remarks No-long-jump zone!!!
6006 */
6007static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6008{
6009 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6010 {
6011 uint32_t uVal = 0;
6012 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6013 AssertRCReturn(rc, rc);
6014
6015 pMixedCtx->eflags.u32 = uVal;
6016 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6017 {
6018 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6019 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6020
6021 pMixedCtx->eflags.Bits.u1VM = 0;
6022 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6023 }
6024
6025 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6026 }
6027 return VINF_SUCCESS;
6028}
6029
6030
6031/**
6032 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6033 * guest-CPU context.
6034 */
6035DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6036{
6037 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6038 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6039 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6040 return rc;
6041}
6042
6043
6044/**
6045 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6046 * from the guest-state area in the VMCS.
6047 *
6048 * @param pVCpu Pointer to the VMCPU.
6049 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6050 * out-of-sync. Make sure to update the required fields
6051 * before using them.
6052 *
6053 * @remarks No-long-jump zone!!!
6054 */
6055static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6056{
6057 uint32_t uIntrState = 0;
6058 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6059 AssertRC(rc);
6060
6061 if (!uIntrState)
6062 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6063 else
6064 {
6065 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
6066 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
6067 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6068 AssertRC(rc);
6069 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6070 AssertRC(rc);
6071
6072 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6073 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6074 }
6075}
6076
6077
6078/**
6079 * Saves the guest's activity state.
6080 *
6081 * @returns VBox status code.
6082 * @param pVCpu Pointer to the VMCPU.
6083 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6084 * out-of-sync. Make sure to update the required fields
6085 * before using them.
6086 *
6087 * @remarks No-long-jump zone!!!
6088 */
6089static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6090{
6091 NOREF(pMixedCtx);
6092 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6093 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6094 return VINF_SUCCESS;
6095}
6096
6097
6098/**
6099 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6100 * the current VMCS into the guest-CPU context.
6101 *
6102 * @returns VBox status code.
6103 * @param pVCpu Pointer to the VMCPU.
6104 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6105 * out-of-sync. Make sure to update the required fields
6106 * before using them.
6107 *
6108 * @remarks No-long-jump zone!!!
6109 */
6110static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6111{
6112 int rc = VINF_SUCCESS;
6113 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6114 {
6115 uint32_t u32Val = 0;
6116 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6117 pMixedCtx->SysEnter.cs = u32Val;
6118 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6119 }
6120
6121 uint64_t u64Val = 0;
6122 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6123 {
6124 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6125 pMixedCtx->SysEnter.eip = u64Val;
6126 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6127 }
6128 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6129 {
6130 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6131 pMixedCtx->SysEnter.esp = u64Val;
6132 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6133 }
6134 return rc;
6135}
6136
6137
6138/**
6139 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6140 * the CPU back into the guest-CPU context.
6141 *
6142 * @returns VBox status code.
6143 * @param pVCpu Pointer to the VMCPU.
6144 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6145 * out-of-sync. Make sure to update the required fields
6146 * before using them.
6147 *
6148 * @remarks No-long-jump zone!!!
6149 */
6150static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6151{
6152#if HC_ARCH_BITS == 64
6153 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6154 {
6155 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6156 VMMRZCallRing3Disable(pVCpu);
6157 HM_DISABLE_PREEMPT_IF_NEEDED();
6158
6159 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6160 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6161 {
6162 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6163 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6164 }
6165
6166 HM_RESTORE_PREEMPT_IF_NEEDED();
6167 VMMRZCallRing3Enable(pVCpu);
6168 }
6169 else
6170 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6171#else
6172 NOREF(pMixedCtx);
6173 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6174#endif
6175
6176 return VINF_SUCCESS;
6177}
6178
6179
6180/**
6181 * Saves the auto load/store'd guest MSRs from the current VMCS into
6182 * the guest-CPU context.
6183 *
6184 * @returns VBox status code.
6185 * @param pVCpu Pointer to the VMCPU.
6186 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6187 * out-of-sync. Make sure to update the required fields
6188 * before using them.
6189 *
6190 * @remarks No-long-jump zone!!!
6191 */
6192static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6193{
6194 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6195 return VINF_SUCCESS;
6196
6197 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6198 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6199 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6200 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6201 {
6202 switch (pMsr->u32Msr)
6203 {
6204 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6205 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6206 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6207 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6208 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6209 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6210 break;
6211
6212 default:
6213 {
6214 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6215 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6216 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6217 }
6218 }
6219 }
6220
6221 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6222 return VINF_SUCCESS;
6223}
6224
6225
6226/**
6227 * Saves the guest control registers from the current VMCS into the guest-CPU
6228 * context.
6229 *
6230 * @returns VBox status code.
6231 * @param pVCpu Pointer to the VMCPU.
6232 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6233 * out-of-sync. Make sure to update the required fields
6234 * before using them.
6235 *
6236 * @remarks No-long-jump zone!!!
6237 */
6238static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6239{
6240 /* Guest CR0. Guest FPU. */
6241 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6242 AssertRCReturn(rc, rc);
6243
6244 /* Guest CR4. */
6245 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6246 AssertRCReturn(rc, rc);
6247
6248 /* Guest CR2 - updated always during the world-switch or in #PF. */
6249 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6250 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6251 {
6252 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6253 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6254
6255 PVM pVM = pVCpu->CTX_SUFF(pVM);
6256 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6257 || ( pVM->hm.s.fNestedPaging
6258 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6259 {
6260 uint64_t u64Val = 0;
6261 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6262 if (pMixedCtx->cr3 != u64Val)
6263 {
6264 CPUMSetGuestCR3(pVCpu, u64Val);
6265 if (VMMRZCallRing3IsEnabled(pVCpu))
6266 {
6267 PGMUpdateCR3(pVCpu, u64Val);
6268 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6269 }
6270 else
6271 {
6272 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6273 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6274 }
6275 }
6276
6277 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6278 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6279 {
6280 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6281 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6282 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6283 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6284
6285 if (VMMRZCallRing3IsEnabled(pVCpu))
6286 {
6287 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6288 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6289 }
6290 else
6291 {
6292 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6293 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6294 }
6295 }
6296 }
6297
6298 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6299 }
6300
6301 /*
6302 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6303 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6304 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6305 *
6306 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6307 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6308 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6309 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6310 *
6311 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6312 */
6313 if (VMMRZCallRing3IsEnabled(pVCpu))
6314 {
6315 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6316 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6317
6318 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6319 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6320
6321 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6322 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6323 }
6324
6325 return rc;
6326}
6327
6328
6329/**
6330 * Reads a guest segment register from the current VMCS into the guest-CPU
6331 * context.
6332 *
6333 * @returns VBox status code.
6334 * @param pVCpu Pointer to the VMCPU.
6335 * @param idxSel Index of the selector in the VMCS.
6336 * @param idxLimit Index of the segment limit in the VMCS.
6337 * @param idxBase Index of the segment base in the VMCS.
6338 * @param idxAccess Index of the access rights of the segment in the VMCS.
6339 * @param pSelReg Pointer to the segment selector.
6340 *
6341 * @remarks No-long-jump zone!!!
6342 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6343 * macro as that takes care of whether to read from the VMCS cache or
6344 * not.
6345 */
6346DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6347 PCPUMSELREG pSelReg)
6348{
6349 NOREF(pVCpu);
6350
6351 uint32_t u32Val = 0;
6352 int rc = VMXReadVmcs32(idxSel, &u32Val);
6353 AssertRCReturn(rc, rc);
6354 pSelReg->Sel = (uint16_t)u32Val;
6355 pSelReg->ValidSel = (uint16_t)u32Val;
6356 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6357
6358 rc = VMXReadVmcs32(idxLimit, &u32Val);
6359 AssertRCReturn(rc, rc);
6360 pSelReg->u32Limit = u32Val;
6361
6362 uint64_t u64Val = 0;
6363 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6364 AssertRCReturn(rc, rc);
6365 pSelReg->u64Base = u64Val;
6366
6367 rc = VMXReadVmcs32(idxAccess, &u32Val);
6368 AssertRCReturn(rc, rc);
6369 pSelReg->Attr.u = u32Val;
6370
6371 /*
6372 * If VT-x marks the segment as unusable, most other bits remain undefined:
6373 * - For CS the L, D and G bits have meaning.
6374 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6375 * - For the remaining data segments no bits are defined.
6376 *
6377 * The present bit and the unusable bit has been observed to be set at the
6378 * same time (the selector was supposed to invalid as we started executing
6379 * a V8086 interrupt in ring-0).
6380 *
6381 * What should be important for the rest of the VBox code, is that the P bit is
6382 * cleared. Some of the other VBox code recognizes the unusable bit, but
6383 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6384 * safe side here, we'll strip off P and other bits we don't care about. If
6385 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6386 *
6387 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6388 */
6389 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6390 {
6391 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6392
6393 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6394 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6395 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6396
6397 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6398#ifdef DEBUG_bird
6399 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6400 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6401 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6402#endif
6403 }
6404 return VINF_SUCCESS;
6405}
6406
6407
6408#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6409# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6410 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6411 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6412#else
6413# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6414 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6415 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6416#endif
6417
6418
6419/**
6420 * Saves the guest segment registers from the current VMCS into the guest-CPU
6421 * context.
6422 *
6423 * @returns VBox status code.
6424 * @param pVCpu Pointer to the VMCPU.
6425 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6426 * out-of-sync. Make sure to update the required fields
6427 * before using them.
6428 *
6429 * @remarks No-long-jump zone!!!
6430 */
6431static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6432{
6433 /* Guest segment registers. */
6434 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6435 {
6436 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6437 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6438 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6439 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6440 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6441 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6442 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6443
6444 /* Restore segment attributes for real-on-v86 mode hack. */
6445 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6446 {
6447 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6448 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6449 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6450 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6451 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6452 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6453 }
6454 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6455 }
6456
6457 return VINF_SUCCESS;
6458}
6459
6460
6461/**
6462 * Saves the guest descriptor table registers and task register from the current
6463 * VMCS into the guest-CPU context.
6464 *
6465 * @returns VBox status code.
6466 * @param pVCpu Pointer to the VMCPU.
6467 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6468 * out-of-sync. Make sure to update the required fields
6469 * before using them.
6470 *
6471 * @remarks No-long-jump zone!!!
6472 */
6473static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6474{
6475 int rc = VINF_SUCCESS;
6476
6477 /* Guest LDTR. */
6478 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6479 {
6480 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6481 AssertRCReturn(rc, rc);
6482 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6483 }
6484
6485 /* Guest GDTR. */
6486 uint64_t u64Val = 0;
6487 uint32_t u32Val = 0;
6488 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6489 {
6490 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6491 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6492 pMixedCtx->gdtr.pGdt = u64Val;
6493 pMixedCtx->gdtr.cbGdt = u32Val;
6494 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6495 }
6496
6497 /* Guest IDTR. */
6498 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6499 {
6500 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6501 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6502 pMixedCtx->idtr.pIdt = u64Val;
6503 pMixedCtx->idtr.cbIdt = u32Val;
6504 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6505 }
6506
6507 /* Guest TR. */
6508 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6509 {
6510 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6511 AssertRCReturn(rc, rc);
6512
6513 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6514 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6515 {
6516 rc = VMXLOCAL_READ_SEG(TR, tr);
6517 AssertRCReturn(rc, rc);
6518 }
6519 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6520 }
6521 return rc;
6522}
6523
6524#undef VMXLOCAL_READ_SEG
6525
6526
6527/**
6528 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6529 * context.
6530 *
6531 * @returns VBox status code.
6532 * @param pVCpu Pointer to the VMCPU.
6533 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6534 * out-of-sync. Make sure to update the required fields
6535 * before using them.
6536 *
6537 * @remarks No-long-jump zone!!!
6538 */
6539static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6540{
6541 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6542 {
6543 if (!pVCpu->hm.s.fUsingHyperDR7)
6544 {
6545 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6546 uint32_t u32Val;
6547 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6548 pMixedCtx->dr[7] = u32Val;
6549 }
6550
6551 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6552 }
6553 return VINF_SUCCESS;
6554}
6555
6556
6557/**
6558 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6559 *
6560 * @returns VBox status code.
6561 * @param pVCpu Pointer to the VMCPU.
6562 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6563 * out-of-sync. Make sure to update the required fields
6564 * before using them.
6565 *
6566 * @remarks No-long-jump zone!!!
6567 */
6568static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6569{
6570 NOREF(pMixedCtx);
6571
6572 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6573 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6574 return VINF_SUCCESS;
6575}
6576
6577
6578/**
6579 * Saves the entire guest state from the currently active VMCS into the
6580 * guest-CPU context. This essentially VMREADs all guest-data.
6581 *
6582 * @returns VBox status code.
6583 * @param pVCpu Pointer to the VMCPU.
6584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6585 * out-of-sync. Make sure to update the required fields
6586 * before using them.
6587 */
6588static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6589{
6590 Assert(pVCpu);
6591 Assert(pMixedCtx);
6592
6593 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6594 return VINF_SUCCESS;
6595
6596 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6597 again on the ring-3 callback path, there is no real need to. */
6598 if (VMMRZCallRing3IsEnabled(pVCpu))
6599 VMMR0LogFlushDisable(pVCpu);
6600 else
6601 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6602 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6603
6604 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6605 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6606
6607 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6608 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6609
6610 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6611 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6612
6613 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6614 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6615
6616 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6617 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6618
6619 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6620 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6621
6622 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6623 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6624
6625 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6626 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6627
6628 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6629 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6630
6631 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6632 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6633
6634 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6635 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6636
6637 if (VMMRZCallRing3IsEnabled(pVCpu))
6638 VMMR0LogFlushEnable(pVCpu);
6639
6640 return rc;
6641}
6642
6643
6644/**
6645 * Check per-VM and per-VCPU force flag actions that require us to go back to
6646 * ring-3 for one reason or another.
6647 *
6648 * @returns VBox status code (information status code included).
6649 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6650 * ring-3.
6651 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6652 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6653 * interrupts)
6654 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6655 * all EMTs to be in ring-3.
6656 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6657 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6658 * to the EM loop.
6659 *
6660 * @param pVM Pointer to the VM.
6661 * @param pVCpu Pointer to the VMCPU.
6662 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6663 * out-of-sync. Make sure to update the required fields
6664 * before using them.
6665 */
6666static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6667{
6668 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6669
6670 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6671 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6672 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6673 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6674 {
6675 /* We need the control registers now, make sure the guest-CPU context is updated. */
6676 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6677 AssertRCReturn(rc3, rc3);
6678
6679 /* Pending HM CR3 sync. */
6680 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6681 {
6682 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6683 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6684 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6685 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6686 }
6687
6688 /* Pending HM PAE PDPEs. */
6689 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6690 {
6691 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6692 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6693 }
6694
6695 /* Pending PGM C3 sync. */
6696 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6697 {
6698 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6699 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6700 if (rc2 != VINF_SUCCESS)
6701 {
6702 AssertRC(rc2);
6703 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6704 return rc2;
6705 }
6706 }
6707
6708 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6709 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6710 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6711 {
6712 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6713 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6714 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6715 return rc2;
6716 }
6717
6718 /* Pending VM request packets, such as hardware interrupts. */
6719 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6720 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6721 {
6722 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6723 return VINF_EM_PENDING_REQUEST;
6724 }
6725
6726 /* Pending PGM pool flushes. */
6727 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6728 {
6729 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6730 return VINF_PGM_POOL_FLUSH_PENDING;
6731 }
6732
6733 /* Pending DMA requests. */
6734 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6735 {
6736 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6737 return VINF_EM_RAW_TO_R3;
6738 }
6739 }
6740
6741 return VINF_SUCCESS;
6742}
6743
6744
6745/**
6746 * Converts any TRPM trap into a pending HM event. This is typically used when
6747 * entering from ring-3 (not longjmp returns).
6748 *
6749 * @param pVCpu Pointer to the VMCPU.
6750 */
6751static int hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6752{
6753 Assert(TRPMHasTrap(pVCpu));
6754 Assert(!pVCpu->hm.s.Event.fPending);
6755
6756 uint8_t uVector;
6757 TRPMEVENT enmTrpmEvent;
6758 RTGCUINT uErrCode;
6759 RTGCUINTPTR GCPtrFaultAddress;
6760 uint8_t cbInstr;
6761
6762 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6763 AssertRC(rc);
6764
6765 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6766 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6767 if (enmTrpmEvent == TRPM_TRAP)
6768 {
6769 switch (uVector)
6770 {
6771 case X86_XCPT_BP:
6772 case X86_XCPT_OF:
6773 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6774 break;
6775
6776 case X86_XCPT_PF:
6777 case X86_XCPT_DF:
6778 case X86_XCPT_TS:
6779 case X86_XCPT_NP:
6780 case X86_XCPT_SS:
6781 case X86_XCPT_GP:
6782 case X86_XCPT_AC:
6783 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6784 /* no break! */
6785 default:
6786 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6787 break;
6788 }
6789 }
6790 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6791 {
6792 if (uVector == X86_XCPT_NMI)
6793 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6794 else
6795 {
6796 uint32_t uEFlags = CPUMGetGuestEFlags(pVCpu);
6797 if (!(uEFlags & X86_EFL_IF))
6798 return VERR_VMX_IPE_5;
6799 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6800 }
6801 }
6802 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6803 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6804 else
6805 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6806
6807 rc = TRPMResetTrap(pVCpu);
6808 AssertRC(rc);
6809 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6810 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6811
6812 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6813 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6814 return VINF_SUCCESS;
6815}
6816
6817
6818/**
6819 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6820 * VT-x to execute any instruction.
6821 *
6822 * @param pvCpu Pointer to the VMCPU.
6823 */
6824static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6825{
6826 Assert(pVCpu->hm.s.Event.fPending);
6827
6828 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6829 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6830 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6831 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6832
6833 /* If a trap was already pending, we did something wrong! */
6834 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6835
6836 TRPMEVENT enmTrapType;
6837 switch (uVectorType)
6838 {
6839 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6840 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6841 enmTrapType = TRPM_HARDWARE_INT;
6842 break;
6843
6844 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6845 enmTrapType = TRPM_SOFTWARE_INT;
6846 break;
6847
6848 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6849 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6850 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6851 enmTrapType = TRPM_TRAP;
6852 break;
6853
6854 default:
6855 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6856 enmTrapType = TRPM_32BIT_HACK;
6857 break;
6858 }
6859
6860 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6861
6862 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6863 AssertRC(rc);
6864
6865 if (fErrorCodeValid)
6866 TRPMSetErrorCode(pVCpu, uErrorCode);
6867
6868 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6869 && uVector == X86_XCPT_PF)
6870 {
6871 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6872 }
6873 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6874 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6875 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6876 {
6877 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6878 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6879 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6880 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6881 }
6882 pVCpu->hm.s.Event.fPending = false;
6883}
6884
6885
6886/**
6887 * Does the necessary state syncing before returning to ring-3 for any reason
6888 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6889 *
6890 * @returns VBox status code.
6891 * @param pVM Pointer to the VM.
6892 * @param pVCpu Pointer to the VMCPU.
6893 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6894 * be out-of-sync. Make sure to update the required
6895 * fields before using them.
6896 * @param fSaveGuestState Whether to save the guest state or not.
6897 *
6898 * @remarks No-long-jmp zone!!!
6899 */
6900static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6901{
6902 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6903 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6904
6905 RTCPUID idCpu = RTMpCpuId();
6906 Log4Func(("HostCpuId=%u\n", idCpu));
6907
6908 /*
6909 * !!! IMPORTANT !!!
6910 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6911 */
6912
6913 /* Save the guest state if necessary. */
6914 if ( fSaveGuestState
6915 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6916 {
6917 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6918 AssertRCReturn(rc, rc);
6919 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6920 }
6921
6922 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6923 if (CPUMIsGuestFPUStateActive(pVCpu))
6924 {
6925 /* We shouldn't reload CR0 without saving it first. */
6926 if (!fSaveGuestState)
6927 {
6928 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6929 AssertRCReturn(rc, rc);
6930 }
6931 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6932 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6933 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6934 }
6935
6936 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6937#ifdef VBOX_STRICT
6938 if (CPUMIsHyperDebugStateActive(pVCpu))
6939 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6940#endif
6941 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6942 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6943 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6944 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6945
6946#if HC_ARCH_BITS == 64
6947 /* Restore host-state bits that VT-x only restores partially. */
6948 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6949 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6950 {
6951 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6952 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6953 }
6954 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6955#endif
6956
6957#if HC_ARCH_BITS == 64
6958 /* Restore the host MSRs as we're leaving VT-x context. */
6959 if ( pVM->hm.s.fAllow64BitGuests
6960 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6961 {
6962 /* We shouldn't reload the guest MSRs without saving it first. */
6963 if (!fSaveGuestState)
6964 {
6965 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6966 AssertRCReturn(rc, rc);
6967 }
6968 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6969 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6970 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6971 }
6972#endif
6973
6974 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6975 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6976
6977 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6978 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6979 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6980 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6981 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6982 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6983 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6984 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6985
6986 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6987
6988 /** @todo This partially defeats the purpose of having preemption hooks.
6989 * The problem is, deregistering the hooks should be moved to a place that
6990 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6991 * context.
6992 */
6993 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6994 {
6995 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6996 AssertRCReturn(rc, rc);
6997
6998 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6999 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7000 }
7001 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7002 NOREF(idCpu);
7003
7004 return VINF_SUCCESS;
7005}
7006
7007
7008/**
7009 * Leaves the VT-x session.
7010 *
7011 * @returns VBox status code.
7012 * @param pVM Pointer to the VM.
7013 * @param pVCpu Pointer to the VMCPU.
7014 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7015 * out-of-sync. Make sure to update the required fields
7016 * before using them.
7017 *
7018 * @remarks No-long-jmp zone!!!
7019 */
7020DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7021{
7022 HM_DISABLE_PREEMPT_IF_NEEDED();
7023 HMVMX_ASSERT_CPU_SAFE();
7024 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7025 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7026
7027 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7028 and done this from the VMXR0ThreadCtxCallback(). */
7029 if (!pVCpu->hm.s.fLeaveDone)
7030 {
7031 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7032 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7033 pVCpu->hm.s.fLeaveDone = true;
7034 }
7035
7036 /*
7037 * !!! IMPORTANT !!!
7038 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7039 */
7040
7041 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7042 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7043 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7044 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7045 VMMR0ThreadCtxHooksDeregister(pVCpu);
7046
7047 /* Leave HM context. This takes care of local init (term). */
7048 int rc = HMR0LeaveCpu(pVCpu);
7049
7050 HM_RESTORE_PREEMPT_IF_NEEDED();
7051
7052 return rc;
7053}
7054
7055
7056/**
7057 * Does the necessary state syncing before doing a longjmp to ring-3.
7058 *
7059 * @returns VBox status code.
7060 * @param pVM Pointer to the VM.
7061 * @param pVCpu Pointer to the VMCPU.
7062 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7063 * out-of-sync. Make sure to update the required fields
7064 * before using them.
7065 *
7066 * @remarks No-long-jmp zone!!!
7067 */
7068DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7069{
7070 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7071}
7072
7073
7074/**
7075 * Take necessary actions before going back to ring-3.
7076 *
7077 * An action requires us to go back to ring-3. This function does the necessary
7078 * steps before we can safely return to ring-3. This is not the same as longjmps
7079 * to ring-3, this is voluntary and prepares the guest so it may continue
7080 * executing outside HM (recompiler/IEM).
7081 *
7082 * @returns VBox status code.
7083 * @param pVM Pointer to the VM.
7084 * @param pVCpu Pointer to the VMCPU.
7085 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7086 * out-of-sync. Make sure to update the required fields
7087 * before using them.
7088 * @param rcExit The reason for exiting to ring-3. Can be
7089 * VINF_VMM_UNKNOWN_RING3_CALL.
7090 */
7091static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7092{
7093 Assert(pVM);
7094 Assert(pVCpu);
7095 Assert(pMixedCtx);
7096 HMVMX_ASSERT_PREEMPT_SAFE();
7097
7098 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7099 {
7100 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7101 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7102 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7103 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7104 }
7105
7106 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7107 VMMRZCallRing3Disable(pVCpu);
7108 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7109
7110 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7111 if (pVCpu->hm.s.Event.fPending)
7112 {
7113 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7114 Assert(!pVCpu->hm.s.Event.fPending);
7115 }
7116
7117 /* Save guest state and restore host state bits. */
7118 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7119 AssertRCReturn(rc, rc);
7120 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7121
7122 /* Sync recompiler state. */
7123 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7124 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7125 | CPUM_CHANGED_LDTR
7126 | CPUM_CHANGED_GDTR
7127 | CPUM_CHANGED_IDTR
7128 | CPUM_CHANGED_TR
7129 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7130 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7131 if ( pVM->hm.s.fNestedPaging
7132 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7133 {
7134 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7135 }
7136
7137 Assert(!pVCpu->hm.s.fClearTrapFlag);
7138
7139 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7140 if (rcExit != VINF_EM_RAW_INTERRUPT)
7141 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7142
7143 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7144
7145 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7146 VMMRZCallRing3RemoveNotification(pVCpu);
7147 VMMRZCallRing3Enable(pVCpu);
7148
7149 return rc;
7150}
7151
7152
7153/**
7154 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7155 * longjump to ring-3 and possibly get preempted.
7156 *
7157 * @returns VBox status code.
7158 * @param pVCpu Pointer to the VMCPU.
7159 * @param enmOperation The operation causing the ring-3 longjump.
7160 * @param pvUser Opaque pointer to the guest-CPU context. The data
7161 * may be out-of-sync. Make sure to update the required
7162 * fields before using them.
7163 */
7164DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7165{
7166 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7167 {
7168 /*
7169 * !!! IMPORTANT !!!
7170 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7171 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7172 */
7173 VMMRZCallRing3RemoveNotification(pVCpu);
7174 VMMRZCallRing3Disable(pVCpu);
7175 HM_DISABLE_PREEMPT_IF_NEEDED();
7176
7177 PVM pVM = pVCpu->CTX_SUFF(pVM);
7178 if (CPUMIsGuestFPUStateActive(pVCpu))
7179 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7180
7181 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7182
7183#if HC_ARCH_BITS == 64
7184 /* Restore host-state bits that VT-x only restores partially. */
7185 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7186 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7187 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7188 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7189
7190 /* Restore the host MSRs as we're leaving VT-x context. */
7191 if ( pVM->hm.s.fAllow64BitGuests
7192 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7193 {
7194 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7195 }
7196#endif
7197 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7198 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7199 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7200 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7201 {
7202 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7203 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7204 }
7205
7206 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7207 VMMR0ThreadCtxHooksDeregister(pVCpu);
7208
7209 HMR0LeaveCpu(pVCpu);
7210 HM_RESTORE_PREEMPT_IF_NEEDED();
7211 return VINF_SUCCESS;
7212 }
7213
7214 Assert(pVCpu);
7215 Assert(pvUser);
7216 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7217 HMVMX_ASSERT_PREEMPT_SAFE();
7218
7219 VMMRZCallRing3Disable(pVCpu);
7220 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7221
7222 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7223 enmOperation));
7224
7225 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7226 AssertRCReturn(rc, rc);
7227
7228 VMMRZCallRing3Enable(pVCpu);
7229 return VINF_SUCCESS;
7230}
7231
7232
7233/**
7234 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7235 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7236 *
7237 * @param pVCpu Pointer to the VMCPU.
7238 */
7239DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7240{
7241 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7242 {
7243 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7244 {
7245 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7246 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7247 AssertRC(rc);
7248 Log4(("Setup interrupt-window exiting\n"));
7249 }
7250 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7251}
7252
7253
7254/**
7255 * Clears the interrupt-window exiting control in the VMCS.
7256 *
7257 * @param pVCpu Pointer to the VMCPU.
7258 */
7259DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7260{
7261 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7262 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7263 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7264 AssertRC(rc);
7265 Log4(("Cleared interrupt-window exiting\n"));
7266}
7267
7268
7269/**
7270 * Evaluates the event to be delivered to the guest and sets it as the pending
7271 * event.
7272 *
7273 * @param pVCpu Pointer to the VMCPU.
7274 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7275 * out-of-sync. Make sure to update the required fields
7276 * before using them.
7277 */
7278static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7279{
7280 Assert(!pVCpu->hm.s.Event.fPending);
7281
7282 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7283 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7284 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7285 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7286
7287 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7288 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7289 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7290 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7291 Assert(!TRPMHasTrap(pVCpu));
7292
7293 /** @todo SMI. SMIs take priority over NMIs. */
7294 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7295 {
7296 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7297 if ( !fBlockMovSS
7298 && !fBlockSti)
7299 {
7300 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7301 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7302 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7303 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7304
7305 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7306 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7307 }
7308 else
7309 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7310 }
7311 /*
7312 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7313 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7314 */
7315 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7316 && !pVCpu->hm.s.fSingleInstruction)
7317 {
7318 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7319 AssertRC(rc);
7320 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7321 if ( !fBlockInt
7322 && !fBlockSti
7323 && !fBlockMovSS)
7324 {
7325 uint8_t u8Interrupt;
7326 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7327 if (RT_SUCCESS(rc))
7328 {
7329 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7330 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7331 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7332
7333 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7334 }
7335 else
7336 {
7337 /** @todo Does this actually happen? If not turn it into an assertion. */
7338 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7339 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7340 }
7341 }
7342 else
7343 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7344 }
7345}
7346
7347
7348/**
7349 * Sets a pending-debug exception to be delivered to the guest if the guest is
7350 * single-stepping.
7351 *
7352 * @param pVCpu Pointer to the VMCPU.
7353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7354 * out-of-sync. Make sure to update the required fields
7355 * before using them.
7356 */
7357DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7358{
7359 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7360 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7361 {
7362 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7363 AssertRC(rc);
7364 }
7365}
7366
7367
7368/**
7369 * Injects any pending events into the guest if the guest is in a state to
7370 * receive them.
7371 *
7372 * @returns VBox status code (informational status codes included).
7373 * @param pVCpu Pointer to the VMCPU.
7374 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7375 * out-of-sync. Make sure to update the required fields
7376 * before using them.
7377 */
7378static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7379{
7380 HMVMX_ASSERT_PREEMPT_SAFE();
7381 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7382
7383 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7384 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7385 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7386 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7387
7388 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7389 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7390 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7391 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7392 Assert(!TRPMHasTrap(pVCpu));
7393
7394 int rc = VINF_SUCCESS;
7395 if (pVCpu->hm.s.Event.fPending)
7396 {
7397 /*
7398 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7399 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7400 * ended up enabling interrupts outside VT-x.
7401 */
7402 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7403 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7404 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7405 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7406 {
7407 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7408 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7409 }
7410#if 1 /* defined(VBOX_STRICT) */ /* Temporarily for debugging. */
7411 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7412 {
7413 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7414 if (fBlockInt)
7415 return VERR_VMX_IPE_4;
7416 Assert(!fBlockSti);
7417 Assert(!fBlockMovSS);
7418 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7419 }
7420 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7421 {
7422 Assert(!fBlockSti);
7423 Assert(!fBlockMovSS);
7424 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7425 }
7426#endif
7427 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7428 (uint8_t)uIntType));
7429 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7430 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7431 AssertRCReturn(rc, rc);
7432
7433 /* Update the interruptibility-state as it could have been changed by
7434 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7435 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7436 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7437
7438#ifdef VBOX_WITH_STATISTICS
7439 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7440 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7441 else
7442 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7443#endif
7444 }
7445
7446 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7447 if ( fBlockSti
7448 || fBlockMovSS)
7449 {
7450 if ( !pVCpu->hm.s.fSingleInstruction
7451 && !DBGFIsStepping(pVCpu))
7452 {
7453 /*
7454 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7455 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7456 * See Intel spec. 27.3.4 "Saving Non-Register State".
7457 */
7458 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7459 AssertRCReturn(rc2, rc2);
7460 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7461 }
7462 else if (pMixedCtx->eflags.Bits.u1TF)
7463 {
7464 /*
7465 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7466 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7467 */
7468 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7469 uIntrState = 0;
7470 }
7471 }
7472
7473 /*
7474 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7475 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7476 */
7477 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7478 AssertRC(rc2);
7479
7480 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7481 NOREF(fBlockMovSS); NOREF(fBlockSti);
7482 return rc;
7483}
7484
7485
7486/**
7487 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7488 *
7489 * @param pVCpu Pointer to the VMCPU.
7490 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7491 * out-of-sync. Make sure to update the required fields
7492 * before using them.
7493 */
7494DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7495{
7496 NOREF(pMixedCtx);
7497 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7498 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7499}
7500
7501
7502/**
7503 * Injects a double-fault (#DF) exception into the VM.
7504 *
7505 * @returns VBox status code (informational status code included).
7506 * @param pVCpu Pointer to the VMCPU.
7507 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7508 * out-of-sync. Make sure to update the required fields
7509 * before using them.
7510 */
7511DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7512{
7513 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7514 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7515 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7516 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7517 puIntrState);
7518}
7519
7520
7521/**
7522 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7523 *
7524 * @param pVCpu Pointer to the VMCPU.
7525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7526 * out-of-sync. Make sure to update the required fields
7527 * before using them.
7528 */
7529DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7530{
7531 NOREF(pMixedCtx);
7532 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7533 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7534 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7535}
7536
7537
7538/**
7539 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7540 *
7541 * @param pVCpu Pointer to the VMCPU.
7542 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7543 * out-of-sync. Make sure to update the required fields
7544 * before using them.
7545 * @param cbInstr The value of RIP that is to be pushed on the guest
7546 * stack.
7547 */
7548DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7549{
7550 NOREF(pMixedCtx);
7551 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7552 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7553 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7554}
7555
7556
7557/**
7558 * Injects a general-protection (#GP) fault into the VM.
7559 *
7560 * @returns VBox status code (informational status code included).
7561 * @param pVCpu Pointer to the VMCPU.
7562 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7563 * out-of-sync. Make sure to update the required fields
7564 * before using them.
7565 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7566 * mode, i.e. in real-mode it's not valid).
7567 * @param u32ErrorCode The error code associated with the #GP.
7568 */
7569DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7570 uint32_t *puIntrState)
7571{
7572 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7573 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7574 if (fErrorCodeValid)
7575 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7576 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7577 puIntrState);
7578}
7579
7580
7581/**
7582 * Sets a general-protection (#GP) exception as pending-for-injection into the
7583 * VM.
7584 *
7585 * @param pVCpu Pointer to the VMCPU.
7586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7587 * out-of-sync. Make sure to update the required fields
7588 * before using them.
7589 * @param u32ErrorCode The error code associated with the #GP.
7590 */
7591DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7592{
7593 NOREF(pMixedCtx);
7594 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7595 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7596 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7597 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7598}
7599
7600
7601/**
7602 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7603 *
7604 * @param pVCpu Pointer to the VMCPU.
7605 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7606 * out-of-sync. Make sure to update the required fields
7607 * before using them.
7608 * @param uVector The software interrupt vector number.
7609 * @param cbInstr The value of RIP that is to be pushed on the guest
7610 * stack.
7611 */
7612DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7613{
7614 NOREF(pMixedCtx);
7615 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7616 if ( uVector == X86_XCPT_BP
7617 || uVector == X86_XCPT_OF)
7618 {
7619 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7620 }
7621 else
7622 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7623 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7624}
7625
7626
7627/**
7628 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7629 * stack.
7630 *
7631 * @returns VBox status code (information status code included).
7632 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7633 * @param pVM Pointer to the VM.
7634 * @param pMixedCtx Pointer to the guest-CPU context.
7635 * @param uValue The value to push to the guest stack.
7636 */
7637DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7638{
7639 /*
7640 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7641 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7642 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7643 */
7644 if (pMixedCtx->sp == 1)
7645 return VINF_EM_RESET;
7646 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7647 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7648 AssertRCReturn(rc, rc);
7649 return rc;
7650}
7651
7652
7653/**
7654 * Injects an event into the guest upon VM-entry by updating the relevant fields
7655 * in the VM-entry area in the VMCS.
7656 *
7657 * @returns VBox status code (informational error codes included).
7658 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7659 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7660 *
7661 * @param pVCpu Pointer to the VMCPU.
7662 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7663 * be out-of-sync. Make sure to update the required
7664 * fields before using them.
7665 * @param u64IntInfo The VM-entry interruption-information field.
7666 * @param cbInstr The VM-entry instruction length in bytes (for
7667 * software interrupts, exceptions and privileged
7668 * software exceptions).
7669 * @param u32ErrCode The VM-entry exception error code.
7670 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7671 * @param puIntrState Pointer to the current guest interruptibility-state.
7672 * This interruptibility-state will be updated if
7673 * necessary. This cannot not be NULL.
7674 *
7675 * @remarks Requires CR0!
7676 * @remarks No-long-jump zone!!!
7677 */
7678static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7679 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7680{
7681 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7682 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7683 Assert(puIntrState);
7684 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7685
7686 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7687 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7688
7689#ifdef VBOX_STRICT
7690 /* Validate the error-code-valid bit for hardware exceptions. */
7691 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7692 {
7693 switch (uVector)
7694 {
7695 case X86_XCPT_PF:
7696 case X86_XCPT_DF:
7697 case X86_XCPT_TS:
7698 case X86_XCPT_NP:
7699 case X86_XCPT_SS:
7700 case X86_XCPT_GP:
7701 case X86_XCPT_AC:
7702 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7703 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7704 /* fallthru */
7705 default:
7706 break;
7707 }
7708 }
7709#endif
7710
7711 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7712 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7713 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7714
7715 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7716
7717 /* We require CR0 to check if the guest is in real-mode. */
7718 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7719 AssertRCReturn(rc, rc);
7720
7721 /*
7722 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7723 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7724 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7725 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7726 */
7727 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7728 {
7729 PVM pVM = pVCpu->CTX_SUFF(pVM);
7730 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7731 {
7732 Assert(PDMVmmDevHeapIsEnabled(pVM));
7733 Assert(pVM->hm.s.vmx.pRealModeTSS);
7734
7735 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7736 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7737 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7738 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7739 AssertRCReturn(rc, rc);
7740 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7741
7742 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7743 const size_t cbIdtEntry = sizeof(X86IDTR16);
7744 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7745 {
7746 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7747 if (uVector == X86_XCPT_DF)
7748 return VINF_EM_RESET;
7749 else if (uVector == X86_XCPT_GP)
7750 {
7751 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7752 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7753 }
7754
7755 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7756 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7757 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7758 }
7759
7760 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7761 uint16_t uGuestIp = pMixedCtx->ip;
7762 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7763 {
7764 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7765 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7766 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7767 }
7768 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7769 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7770
7771 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7772 X86IDTR16 IdtEntry;
7773 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7774 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7775 AssertRCReturn(rc, rc);
7776
7777 /* Construct the stack frame for the interrupt/exception handler. */
7778 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7779 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7780 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7781 AssertRCReturn(rc, rc);
7782
7783 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7784 if (rc == VINF_SUCCESS)
7785 {
7786 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7787 pMixedCtx->rip = IdtEntry.offSel;
7788 pMixedCtx->cs.Sel = IdtEntry.uSel;
7789 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7790 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7791 && uVector == X86_XCPT_PF)
7792 {
7793 pMixedCtx->cr2 = GCPtrFaultAddress;
7794 }
7795
7796 /* If any other guest-state bits are changed here, make sure to update
7797 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7798 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7799 | HM_CHANGED_GUEST_RIP
7800 | HM_CHANGED_GUEST_RFLAGS
7801 | HM_CHANGED_GUEST_RSP);
7802
7803 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7804 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7805 {
7806 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7807 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7808 Log4(("Clearing inhibition due to STI.\n"));
7809 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7810 }
7811 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7812
7813 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7814 it, if we are returning to ring-3 before executing guest code. */
7815 pVCpu->hm.s.Event.fPending = false;
7816 }
7817 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7818 return rc;
7819 }
7820 else
7821 {
7822 /*
7823 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7824 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7825 */
7826 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7827 }
7828 }
7829
7830 /* Validate. */
7831 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7832 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7833 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7834
7835 /* Inject. */
7836 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7837 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7838 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7839 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7840
7841 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7842 && uVector == X86_XCPT_PF)
7843 {
7844 pMixedCtx->cr2 = GCPtrFaultAddress;
7845 }
7846
7847 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7848 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7849
7850 AssertRCReturn(rc, rc);
7851 return rc;
7852}
7853
7854
7855/**
7856 * Clears the interrupt-window exiting control in the VMCS and if necessary
7857 * clears the current event in the VMCS as well.
7858 *
7859 * @returns VBox status code.
7860 * @param pVCpu Pointer to the VMCPU.
7861 *
7862 * @remarks Use this function only to clear events that have not yet been
7863 * delivered to the guest but are injected in the VMCS!
7864 * @remarks No-long-jump zone!!!
7865 */
7866static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7867{
7868 int rc;
7869 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7870
7871 /* Clear interrupt-window exiting control. */
7872 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7873 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7874
7875 if (!pVCpu->hm.s.Event.fPending)
7876 return;
7877
7878#ifdef VBOX_STRICT
7879 uint32_t u32EntryInfo;
7880 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7881 AssertRC(rc);
7882 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7883#endif
7884
7885 /* Clear the entry-interruption field (including the valid bit). */
7886 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7887 AssertRC(rc);
7888
7889 /* Clear the pending debug exception field. */
7890 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7891 AssertRC(rc);
7892
7893 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7894 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7895}
7896
7897
7898/**
7899 * Enters the VT-x session.
7900 *
7901 * @returns VBox status code.
7902 * @param pVM Pointer to the VM.
7903 * @param pVCpu Pointer to the VMCPU.
7904 * @param pCpu Pointer to the CPU info struct.
7905 */
7906VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7907{
7908 AssertPtr(pVM);
7909 AssertPtr(pVCpu);
7910 Assert(pVM->hm.s.vmx.fSupported);
7911 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7912 NOREF(pCpu); NOREF(pVM);
7913
7914 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7915 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7916
7917#ifdef VBOX_STRICT
7918 /* Make sure we're in VMX root mode. */
7919 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7920 if (!(u32HostCR4 & X86_CR4_VMXE))
7921 {
7922 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7923 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7924 }
7925#endif
7926
7927 /*
7928 * Load the VCPU's VMCS as the current (and active) one.
7929 */
7930 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7931 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7932 if (RT_FAILURE(rc))
7933 return rc;
7934
7935 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7936 pVCpu->hm.s.fLeaveDone = false;
7937 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7938
7939 return VINF_SUCCESS;
7940}
7941
7942
7943/**
7944 * The thread-context callback (only on platforms which support it).
7945 *
7946 * @param enmEvent The thread-context event.
7947 * @param pVCpu Pointer to the VMCPU.
7948 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7949 * @thread EMT(pVCpu)
7950 */
7951VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7952{
7953 NOREF(fGlobalInit);
7954
7955 switch (enmEvent)
7956 {
7957 case RTTHREADCTXEVENT_PREEMPTING:
7958 {
7959 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7960 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7961 VMCPU_ASSERT_EMT(pVCpu);
7962
7963 PVM pVM = pVCpu->CTX_SUFF(pVM);
7964 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7965
7966 /* No longjmps (logger flushes, locks) in this fragile context. */
7967 VMMRZCallRing3Disable(pVCpu);
7968 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7969
7970 /*
7971 * Restore host-state (FPU, debug etc.)
7972 */
7973 if (!pVCpu->hm.s.fLeaveDone)
7974 {
7975 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7976 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7977 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7978 pVCpu->hm.s.fLeaveDone = true;
7979 }
7980
7981 /* Leave HM context, takes care of local init (term). */
7982 int rc = HMR0LeaveCpu(pVCpu);
7983 AssertRC(rc); NOREF(rc);
7984
7985 /* Restore longjmp state. */
7986 VMMRZCallRing3Enable(pVCpu);
7987 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7988 break;
7989 }
7990
7991 case RTTHREADCTXEVENT_RESUMED:
7992 {
7993 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7994 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7995 VMCPU_ASSERT_EMT(pVCpu);
7996
7997 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7998 VMMRZCallRing3Disable(pVCpu);
7999 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8000
8001 /* Initialize the bare minimum state required for HM. This takes care of
8002 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8003 int rc = HMR0EnterCpu(pVCpu);
8004 AssertRC(rc);
8005 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8006
8007 /* Load the active VMCS as the current one. */
8008 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8009 {
8010 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8011 AssertRC(rc); NOREF(rc);
8012 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8013 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8014 }
8015 pVCpu->hm.s.fLeaveDone = false;
8016
8017 /* Restore longjmp state. */
8018 VMMRZCallRing3Enable(pVCpu);
8019 break;
8020 }
8021
8022 default:
8023 break;
8024 }
8025}
8026
8027
8028/**
8029 * Saves the host state in the VMCS host-state.
8030 * Sets up the VM-exit MSR-load area.
8031 *
8032 * The CPU state will be loaded from these fields on every successful VM-exit.
8033 *
8034 * @returns VBox status code.
8035 * @param pVM Pointer to the VM.
8036 * @param pVCpu Pointer to the VMCPU.
8037 *
8038 * @remarks No-long-jump zone!!!
8039 */
8040static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8041{
8042 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8043
8044 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8045 return VINF_SUCCESS;
8046
8047 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8049
8050 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8052
8053 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8055
8056 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8057 return rc;
8058}
8059
8060
8061/**
8062 * Saves the host state in the VMCS host-state.
8063 *
8064 * @returns VBox status code.
8065 * @param pVM Pointer to the VM.
8066 * @param pVCpu Pointer to the VMCPU.
8067 *
8068 * @remarks No-long-jump zone!!!
8069 */
8070VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8071{
8072 AssertPtr(pVM);
8073 AssertPtr(pVCpu);
8074
8075 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8076
8077 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8078 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8079 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8080 return hmR0VmxSaveHostState(pVM, pVCpu);
8081}
8082
8083
8084/**
8085 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8086 * loaded from these fields on every successful VM-entry.
8087 *
8088 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8089 * Sets up the VM-entry controls.
8090 * Sets up the appropriate VMX non-root function to execute guest code based on
8091 * the guest CPU mode.
8092 *
8093 * @returns VBox status code.
8094 * @param pVM Pointer to the VM.
8095 * @param pVCpu Pointer to the VMCPU.
8096 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8097 * out-of-sync. Make sure to update the required fields
8098 * before using them.
8099 *
8100 * @remarks No-long-jump zone!!!
8101 */
8102static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8103{
8104 AssertPtr(pVM);
8105 AssertPtr(pVCpu);
8106 AssertPtr(pMixedCtx);
8107 HMVMX_ASSERT_PREEMPT_SAFE();
8108
8109#ifdef LOG_ENABLED
8110 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8111 * probably not initialized yet? Anyway this will do for now.
8112 *
8113 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8114 * interface and disable ring-3 calls when thread-context hooks are not
8115 * available. */
8116 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8117 VMMR0LogFlushDisable(pVCpu);
8118#endif
8119
8120 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8121
8122 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8123
8124 /* Determine real-on-v86 mode. */
8125 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8126 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8127 && CPUMIsGuestInRealModeEx(pMixedCtx))
8128 {
8129 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8130 }
8131
8132 /*
8133 * Load the guest-state into the VMCS.
8134 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8135 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8136 */
8137 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8138 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8139
8140 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8141 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8142 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8143
8144 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8145 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8146 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8147
8148 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8149 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8150
8151 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8152 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8153
8154 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8155 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8156 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8157
8158 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8159 determine we don't have to swap EFER after all. */
8160 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8161 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8162
8163 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8164 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8165
8166 /*
8167 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8168 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8169 */
8170 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8171 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8172
8173 /* Clear any unused and reserved bits. */
8174 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8175
8176#ifdef LOG_ENABLED
8177 /* Only reenable log-flushing if the caller has it enabled. */
8178 if (!fCallerDisabledLogFlush)
8179 VMMR0LogFlushEnable(pVCpu);
8180#endif
8181
8182 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8183 return rc;
8184}
8185
8186
8187/**
8188 * Loads the state shared between the host and guest into the VMCS.
8189 *
8190 * @param pVM Pointer to the VM.
8191 * @param pVCpu Pointer to the VMCPU.
8192 * @param pCtx Pointer to the guest-CPU context.
8193 *
8194 * @remarks No-long-jump zone!!!
8195 */
8196static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8197{
8198 NOREF(pVM);
8199
8200 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8201 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8202
8203 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8204 {
8205 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8206 AssertRC(rc);
8207 }
8208
8209 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8210 {
8211 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8212 AssertRC(rc);
8213
8214 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8215 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8216 {
8217 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8218 AssertRC(rc);
8219 }
8220 }
8221
8222 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8223 {
8224#if HC_ARCH_BITS == 64
8225 if (pVM->hm.s.fAllow64BitGuests)
8226 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8227#endif
8228 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8229 }
8230
8231 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8232 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8233}
8234
8235
8236/**
8237 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8238 *
8239 * @param pVM Pointer to the VM.
8240 * @param pVCpu Pointer to the VMCPU.
8241 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8242 * out-of-sync. Make sure to update the required fields
8243 * before using them.
8244 */
8245DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8246{
8247 HMVMX_ASSERT_PREEMPT_SAFE();
8248
8249 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8250#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8251 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8252#endif
8253
8254 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8255 {
8256 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8257 AssertRC(rc);
8258 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8259 }
8260 else if (HMCPU_CF_VALUE(pVCpu))
8261 {
8262 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8263 AssertRC(rc);
8264 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8265 }
8266
8267 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8268 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8269 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8270 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8271}
8272
8273
8274/**
8275 * Does the preparations before executing guest code in VT-x.
8276 *
8277 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8278 * recompiler. We must be cautious what we do here regarding committing
8279 * guest-state information into the VMCS assuming we assuredly execute the
8280 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8281 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8282 * so that the recompiler can (and should) use them when it resumes guest
8283 * execution. Otherwise such operations must be done when we can no longer
8284 * exit to ring-3.
8285 *
8286 * @returns Strict VBox status code.
8287 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8288 * have been disabled.
8289 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8290 * double-fault into the guest.
8291 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8292 *
8293 * @param pVM Pointer to the VM.
8294 * @param pVCpu Pointer to the VMCPU.
8295 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8296 * out-of-sync. Make sure to update the required fields
8297 * before using them.
8298 * @param pVmxTransient Pointer to the VMX transient structure.
8299 */
8300static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8301{
8302 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8303
8304#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8305 PGMRZDynMapFlushAutoSet(pVCpu);
8306#endif
8307
8308 /* Check force flag actions that might require us to go back to ring-3. */
8309 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8310 if (rc != VINF_SUCCESS)
8311 return rc;
8312
8313#ifndef IEM_VERIFICATION_MODE_FULL
8314 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8315 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8316 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8317 {
8318 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8319 RTGCPHYS GCPhysApicBase;
8320 GCPhysApicBase = pMixedCtx->msrApicBase;
8321 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8322
8323 /* Unalias any existing mapping. */
8324 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8325 AssertRCReturn(rc, rc);
8326
8327 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8328 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8329 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8330 AssertRCReturn(rc, rc);
8331
8332 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8333 }
8334#endif /* !IEM_VERIFICATION_MODE_FULL */
8335
8336 /*
8337 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8338 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8339 */
8340 if (TRPMHasTrap(pVCpu))
8341 {
8342 rc = hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8343 if (RT_FAILURE(rc))
8344 return rc;
8345 }
8346 else if (!pVCpu->hm.s.Event.fPending)
8347 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8348
8349 /*
8350 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8351 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8352 */
8353 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8354 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8355 {
8356 //Assert(rc == VINF_EM_RESET);
8357 return rc;
8358 }
8359
8360 /*
8361 * Load the guest state bits, we can handle longjmps/getting preempted here.
8362 *
8363 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8364 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8365 * Hence, this needs to be done -after- injection of events.
8366 */
8367 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8368
8369 /*
8370 * No longjmps to ring-3 from this point on!!!
8371 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8372 * This also disables flushing of the R0-logger instance (if any).
8373 */
8374 VMMRZCallRing3Disable(pVCpu);
8375
8376 /*
8377 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8378 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8379 *
8380 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8381 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8382 *
8383 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8384 * executing guest code.
8385 */
8386 pVmxTransient->uEflags = ASMIntDisableFlags();
8387 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8388 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8389 {
8390 hmR0VmxClearEventVmcs(pVCpu);
8391 ASMSetFlags(pVmxTransient->uEflags);
8392 VMMRZCallRing3Enable(pVCpu);
8393 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8394 return VINF_EM_RAW_TO_R3;
8395 }
8396
8397 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8398 {
8399 hmR0VmxClearEventVmcs(pVCpu);
8400 ASMSetFlags(pVmxTransient->uEflags);
8401 VMMRZCallRing3Enable(pVCpu);
8402 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8403 return VINF_EM_RAW_INTERRUPT;
8404 }
8405
8406 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8407 pVCpu->hm.s.Event.fPending = false;
8408
8409 return VINF_SUCCESS;
8410}
8411
8412
8413/**
8414 * Prepares to run guest code in VT-x and we've committed to doing so. This
8415 * means there is no backing out to ring-3 or anywhere else at this
8416 * point.
8417 *
8418 * @param pVM Pointer to the VM.
8419 * @param pVCpu Pointer to the VMCPU.
8420 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8421 * out-of-sync. Make sure to update the required fields
8422 * before using them.
8423 * @param pVmxTransient Pointer to the VMX transient structure.
8424 *
8425 * @remarks Called with preemption disabled.
8426 * @remarks No-long-jump zone!!!
8427 */
8428static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8429{
8430 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8431 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8432 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8433
8434 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8435 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8436
8437#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8438 if (!CPUMIsGuestFPUStateActive(pVCpu))
8439 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8440 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8441#endif
8442
8443 if ( pVCpu->hm.s.fUseGuestFpu
8444 && !CPUMIsGuestFPUStateActive(pVCpu))
8445 {
8446 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8447 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8448 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8449 }
8450
8451 /*
8452 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8453 */
8454 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8455 && pVCpu->hm.s.vmx.cMsrs > 0)
8456 {
8457 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8458 }
8459
8460 /*
8461 * Load the host state bits as we may've been preempted (only happens when
8462 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8463 */
8464 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8465 {
8466 /* This ASSUMES that pfnStartVM has been set up already. */
8467 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8468 AssertRC(rc);
8469 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8470 }
8471 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8472
8473 /*
8474 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8475 */
8476 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8477 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8478 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8479
8480 /* Store status of the shared guest-host state at the time of VM-entry. */
8481#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8482 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8483 {
8484 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8485 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8486 }
8487 else
8488#endif
8489 {
8490 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8491 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8492 }
8493 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8494
8495 /*
8496 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8497 */
8498 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8499 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8500
8501 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8502 RTCPUID idCurrentCpu = pCpu->idCpu;
8503 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8504 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8505 {
8506 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8507 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8508 }
8509
8510 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8511 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8512 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8513 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8514
8515 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8516
8517 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8518 to start executing. */
8519
8520 /*
8521 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8522 */
8523 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8524 {
8525 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8526 {
8527 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8528 AssertRC(rc2);
8529 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8530 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8531 true /* fUpdateHostMsr */);
8532 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8533 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8534 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8535 }
8536 else
8537 {
8538 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8539 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8540 }
8541 }
8542
8543#ifdef VBOX_STRICT
8544 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8545 hmR0VmxCheckHostEferMsr(pVCpu);
8546 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8547#endif
8548#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8549 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8550 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8551 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8552#endif
8553}
8554
8555
8556/**
8557 * Performs some essential restoration of state after running guest code in
8558 * VT-x.
8559 *
8560 * @param pVM Pointer to the VM.
8561 * @param pVCpu Pointer to the VMCPU.
8562 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8563 * out-of-sync. Make sure to update the required fields
8564 * before using them.
8565 * @param pVmxTransient Pointer to the VMX transient structure.
8566 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8567 *
8568 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8569 *
8570 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8571 * unconditionally when it is safe to do so.
8572 */
8573static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8574{
8575 NOREF(pVM);
8576
8577 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8578
8579 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8580 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8581 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8582 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8583 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8584
8585 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8586 {
8587 /** @todo Find a way to fix hardcoding a guestimate. */
8588 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8589 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8590 }
8591
8592 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8593 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8594 Assert(!(ASMGetFlags() & X86_EFL_IF));
8595 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8596
8597#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8598 if (CPUMIsGuestFPUStateActive(pVCpu))
8599 {
8600 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8601 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8602 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8603 }
8604#endif
8605
8606#if HC_ARCH_BITS == 64
8607 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8608#endif
8609 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8610#ifdef VBOX_STRICT
8611 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8612#endif
8613 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8614 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8615
8616 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8617 uint32_t uExitReason;
8618 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8619 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8620 AssertRC(rc);
8621 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8622 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8623
8624 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8625 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8626 {
8627 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8628 pVmxTransient->fVMEntryFailed));
8629 return;
8630 }
8631
8632 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8633 {
8634 /* Update the guest interruptibility-state from the VMCS. */
8635 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8636
8637#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8638 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8639 AssertRC(rc);
8640#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8641 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8642 AssertRC(rc);
8643#endif
8644
8645 /*
8646 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8647 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8648 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8649 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8650 */
8651 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8652 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8653 {
8654 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8655 AssertRC(rc);
8656 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8657 }
8658 }
8659}
8660
8661
8662/**
8663 * Runs the guest code using VT-x the normal way.
8664 *
8665 * @returns VBox status code.
8666 * @param pVM Pointer to the VM.
8667 * @param pVCpu Pointer to the VMCPU.
8668 * @param pCtx Pointer to the guest-CPU context.
8669 *
8670 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8671 */
8672static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8673{
8674 VMXTRANSIENT VmxTransient;
8675 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8676 int rc = VERR_INTERNAL_ERROR_5;
8677 uint32_t cLoops = 0;
8678
8679 for (;; cLoops++)
8680 {
8681 Assert(!HMR0SuspendPending());
8682 HMVMX_ASSERT_CPU_SAFE();
8683
8684 /* Preparatory work for running guest code, this may force us to return
8685 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8686 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8687 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8688 if (rc != VINF_SUCCESS)
8689 break;
8690
8691 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8692 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8693 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8694
8695 /* Restore any residual host-state and save any bits shared between host
8696 and guest into the guest-CPU state. Re-enables interrupts! */
8697 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8698
8699 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8700 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8701 {
8702 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8703 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8704 return rc;
8705 }
8706
8707 /* Handle the VM-exit. */
8708 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8709 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8710 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8711 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8712 HMVMX_START_EXIT_DISPATCH_PROF();
8713#ifdef HMVMX_USE_FUNCTION_TABLE
8714 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8715#else
8716 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8717#endif
8718 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8719 if (rc != VINF_SUCCESS)
8720 break;
8721 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8722 {
8723 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8724 rc = VINF_EM_RAW_INTERRUPT;
8725 break;
8726 }
8727 }
8728
8729 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8730 return rc;
8731}
8732
8733
8734/**
8735 * Single steps guest code using VT-x.
8736 *
8737 * @returns VBox status code.
8738 * @param pVM Pointer to the VM.
8739 * @param pVCpu Pointer to the VMCPU.
8740 * @param pCtx Pointer to the guest-CPU context.
8741 *
8742 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8743 */
8744static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8745{
8746 VMXTRANSIENT VmxTransient;
8747 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8748 int rc = VERR_INTERNAL_ERROR_5;
8749 uint32_t cLoops = 0;
8750 uint16_t uCsStart = pCtx->cs.Sel;
8751 uint64_t uRipStart = pCtx->rip;
8752
8753 for (;; cLoops++)
8754 {
8755 Assert(!HMR0SuspendPending());
8756 HMVMX_ASSERT_CPU_SAFE();
8757
8758 /* Preparatory work for running guest code, this may force us to return
8759 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8760 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8761 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8762 if (rc != VINF_SUCCESS)
8763 break;
8764
8765 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8766 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8767 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8768
8769 /* Restore any residual host-state and save any bits shared between host
8770 and guest into the guest-CPU state. Re-enables interrupts! */
8771 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8772
8773 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8774 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8775 {
8776 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8777 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8778 return rc;
8779 }
8780
8781 /* Handle the VM-exit. */
8782 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8783 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8784 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8785 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8786 HMVMX_START_EXIT_DISPATCH_PROF();
8787#ifdef HMVMX_USE_FUNCTION_TABLE
8788 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8789#else
8790 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8791#endif
8792 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8793 if (rc != VINF_SUCCESS)
8794 break;
8795 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8796 {
8797 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8798 rc = VINF_EM_RAW_INTERRUPT;
8799 break;
8800 }
8801
8802 /*
8803 * Did the RIP change, if so, consider it a single step.
8804 * Otherwise, make sure one of the TFs gets set.
8805 */
8806 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8807 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8808 AssertRCReturn(rc2, rc2);
8809 if ( pCtx->rip != uRipStart
8810 || pCtx->cs.Sel != uCsStart)
8811 {
8812 rc = VINF_EM_DBG_STEPPED;
8813 break;
8814 }
8815 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8816 }
8817
8818 /*
8819 * Clear the X86_EFL_TF if necessary.
8820 */
8821 if (pVCpu->hm.s.fClearTrapFlag)
8822 {
8823 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8824 AssertRCReturn(rc2, rc2);
8825 pVCpu->hm.s.fClearTrapFlag = false;
8826 pCtx->eflags.Bits.u1TF = 0;
8827 }
8828 /** @todo there seems to be issues with the resume flag when the monitor trap
8829 * flag is pending without being used. Seen early in bios init when
8830 * accessing APIC page in protected mode. */
8831
8832 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8833 return rc;
8834}
8835
8836
8837/**
8838 * Runs the guest code using VT-x.
8839 *
8840 * @returns VBox status code.
8841 * @param pVM Pointer to the VM.
8842 * @param pVCpu Pointer to the VMCPU.
8843 * @param pCtx Pointer to the guest-CPU context.
8844 */
8845VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8846{
8847 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8848 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8849 HMVMX_ASSERT_PREEMPT_SAFE();
8850
8851 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8852
8853 int rc;
8854 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8855 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8856 else
8857 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8858
8859 if (rc == VERR_EM_INTERPRETER)
8860 rc = VINF_EM_RAW_EMULATE_INSTR;
8861 else if (rc == VINF_EM_RESET)
8862 rc = VINF_EM_TRIPLE_FAULT;
8863
8864 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8865 if (RT_FAILURE(rc2))
8866 {
8867 pVCpu->hm.s.u32HMError = rc;
8868 rc = rc2;
8869 }
8870 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8871 return rc;
8872}
8873
8874
8875#ifndef HMVMX_USE_FUNCTION_TABLE
8876DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8877{
8878#ifdef DEBUG_ramshankar
8879# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8880# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8881#endif
8882 int rc;
8883 switch (rcReason)
8884 {
8885 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8886 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8887 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8888 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8889 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8890 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8891 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8892 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8893 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8894 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8895 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8896 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8897 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8898 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8899 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8900 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8901 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8902 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8903 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8904 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8905 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8906 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8907 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8908 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8909 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8910 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8911 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8912 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8913 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8914 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8915 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8916 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8917 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8918
8919 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8920 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8921 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8922 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8923 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8924 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8925 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8926 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8927 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8928
8929 case VMX_EXIT_VMCALL:
8930 case VMX_EXIT_VMCLEAR:
8931 case VMX_EXIT_VMLAUNCH:
8932 case VMX_EXIT_VMPTRLD:
8933 case VMX_EXIT_VMPTRST:
8934 case VMX_EXIT_VMREAD:
8935 case VMX_EXIT_VMRESUME:
8936 case VMX_EXIT_VMWRITE:
8937 case VMX_EXIT_VMXOFF:
8938 case VMX_EXIT_VMXON:
8939 case VMX_EXIT_INVEPT:
8940 case VMX_EXIT_INVVPID:
8941 case VMX_EXIT_VMFUNC:
8942 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8943 break;
8944 default:
8945 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8946 break;
8947 }
8948 return rc;
8949}
8950#endif
8951
8952#ifdef DEBUG
8953/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8954# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8955 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8956
8957# define HMVMX_ASSERT_PREEMPT_CPUID() \
8958 do \
8959 { \
8960 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8961 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8962 } while (0)
8963
8964# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8965 do { \
8966 AssertPtr(pVCpu); \
8967 AssertPtr(pMixedCtx); \
8968 AssertPtr(pVmxTransient); \
8969 Assert(pVmxTransient->fVMEntryFailed == false); \
8970 Assert(ASMIntAreEnabled()); \
8971 HMVMX_ASSERT_PREEMPT_SAFE(); \
8972 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8973 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8974 HMVMX_ASSERT_PREEMPT_SAFE(); \
8975 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8976 HMVMX_ASSERT_PREEMPT_CPUID(); \
8977 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8978 } while (0)
8979
8980# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8981 do { \
8982 Log4Func(("\n")); \
8983 } while (0)
8984#else /* Release builds */
8985# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8986 do { \
8987 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8988 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8989 } while (0)
8990# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8991#endif
8992
8993
8994/**
8995 * Advances the guest RIP after reading it from the VMCS.
8996 *
8997 * @returns VBox status code.
8998 * @param pVCpu Pointer to the VMCPU.
8999 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9000 * out-of-sync. Make sure to update the required fields
9001 * before using them.
9002 * @param pVmxTransient Pointer to the VMX transient structure.
9003 *
9004 * @remarks No-long-jump zone!!!
9005 */
9006DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9007{
9008 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9009 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9010 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9011 AssertRCReturn(rc, rc);
9012
9013 pMixedCtx->rip += pVmxTransient->cbInstr;
9014 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9015
9016 /*
9017 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9018 * pending debug exception field as it takes care of priority of events.
9019 *
9020 * See Intel spec. 32.2.1 "Debug Exceptions".
9021 */
9022 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9023
9024 return rc;
9025}
9026
9027
9028/**
9029 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9030 * and update error record fields accordingly.
9031 *
9032 * @return VMX_IGS_* return codes.
9033 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9034 * wrong with the guest state.
9035 *
9036 * @param pVM Pointer to the VM.
9037 * @param pVCpu Pointer to the VMCPU.
9038 * @param pCtx Pointer to the guest-CPU state.
9039 *
9040 * @remarks This function assumes our cache of the VMCS controls
9041 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9042 */
9043static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9044{
9045#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9046#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9047 uError = (err); \
9048 break; \
9049 } else do { } while (0)
9050
9051 int rc;
9052 uint32_t uError = VMX_IGS_ERROR;
9053 uint32_t u32Val;
9054 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9055
9056 do
9057 {
9058 /*
9059 * CR0.
9060 */
9061 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9062 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9063 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9064 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9065 if (fUnrestrictedGuest)
9066 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9067
9068 uint32_t u32GuestCR0;
9069 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9070 AssertRCBreak(rc);
9071 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9072 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9073 if ( !fUnrestrictedGuest
9074 && (u32GuestCR0 & X86_CR0_PG)
9075 && !(u32GuestCR0 & X86_CR0_PE))
9076 {
9077 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9078 }
9079
9080 /*
9081 * CR4.
9082 */
9083 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9084 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9085
9086 uint32_t u32GuestCR4;
9087 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9088 AssertRCBreak(rc);
9089 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9090 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9091
9092 /*
9093 * IA32_DEBUGCTL MSR.
9094 */
9095 uint64_t u64Val;
9096 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9097 AssertRCBreak(rc);
9098 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9099 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9100 {
9101 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9102 }
9103 uint64_t u64DebugCtlMsr = u64Val;
9104
9105#ifdef VBOX_STRICT
9106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9107 AssertRCBreak(rc);
9108 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9109#endif
9110 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9111
9112 /*
9113 * RIP and RFLAGS.
9114 */
9115 uint32_t u32Eflags;
9116#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9117 if (HMVMX_IS_64BIT_HOST_MODE())
9118 {
9119 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9120 AssertRCBreak(rc);
9121 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9122 if ( !fLongModeGuest
9123 || !pCtx->cs.Attr.n.u1Long)
9124 {
9125 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9126 }
9127 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9128 * must be identical if the "IA-32e mode guest" VM-entry
9129 * control is 1 and CS.L is 1. No check applies if the
9130 * CPU supports 64 linear-address bits. */
9131
9132 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9133 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9134 AssertRCBreak(rc);
9135 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9136 VMX_IGS_RFLAGS_RESERVED);
9137 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9138 u32Eflags = u64Val;
9139 }
9140 else
9141#endif
9142 {
9143 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9144 AssertRCBreak(rc);
9145 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9146 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9147 }
9148
9149 if ( fLongModeGuest
9150 || ( fUnrestrictedGuest
9151 && !(u32GuestCR0 & X86_CR0_PE)))
9152 {
9153 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9154 }
9155
9156 uint32_t u32EntryInfo;
9157 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9158 AssertRCBreak(rc);
9159 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9160 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9161 {
9162 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9163 }
9164
9165 /*
9166 * 64-bit checks.
9167 */
9168#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9169 if (HMVMX_IS_64BIT_HOST_MODE())
9170 {
9171 if ( fLongModeGuest
9172 && !fUnrestrictedGuest)
9173 {
9174 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9175 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9176 }
9177
9178 if ( !fLongModeGuest
9179 && (u32GuestCR4 & X86_CR4_PCIDE))
9180 {
9181 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9182 }
9183
9184 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9185 * 51:32 beyond the processor's physical-address width are 0. */
9186
9187 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9188 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9189 {
9190 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9191 }
9192
9193 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9194 AssertRCBreak(rc);
9195 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9196
9197 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9198 AssertRCBreak(rc);
9199 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9200 }
9201#endif
9202
9203 /*
9204 * PERF_GLOBAL MSR.
9205 */
9206 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9207 {
9208 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9209 AssertRCBreak(rc);
9210 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9211 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9212 }
9213
9214 /*
9215 * PAT MSR.
9216 */
9217 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9218 {
9219 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9220 AssertRCBreak(rc);
9221 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9222 for (unsigned i = 0; i < 8; i++)
9223 {
9224 uint8_t u8Val = (u64Val & 0x7);
9225 if ( u8Val != 0 /* UC */
9226 || u8Val != 1 /* WC */
9227 || u8Val != 4 /* WT */
9228 || u8Val != 5 /* WP */
9229 || u8Val != 6 /* WB */
9230 || u8Val != 7 /* UC- */)
9231 {
9232 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9233 }
9234 u64Val >>= 3;
9235 }
9236 }
9237
9238 /*
9239 * EFER MSR.
9240 */
9241 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9242 {
9243 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9244 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9245 AssertRCBreak(rc);
9246 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9247 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9248 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9249 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9250 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9251 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9252 VMX_IGS_EFER_LMA_PG_MISMATCH);
9253 }
9254
9255 /*
9256 * Segment registers.
9257 */
9258 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9259 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9260 if (!(u32Eflags & X86_EFL_VM))
9261 {
9262 /* CS */
9263 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9264 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9265 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9266 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9267 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9268 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9269 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9270 /* CS cannot be loaded with NULL in protected mode. */
9271 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9272 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9273 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9274 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9275 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9276 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9277 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9278 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9279 else
9280 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9281
9282 /* SS */
9283 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9284 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9285 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9286 if ( !(pCtx->cr0 & X86_CR0_PE)
9287 || pCtx->cs.Attr.n.u4Type == 3)
9288 {
9289 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9290 }
9291 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9292 {
9293 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9294 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9295 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9296 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9297 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9298 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9299 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9300 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9301 }
9302
9303 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9304 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9305 {
9306 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9307 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9308 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9309 || pCtx->ds.Attr.n.u4Type > 11
9310 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9311 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9312 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9313 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9314 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9315 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9316 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9317 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9318 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9319 }
9320 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9321 {
9322 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9323 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9324 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9325 || pCtx->es.Attr.n.u4Type > 11
9326 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9327 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9328 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9329 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9330 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9331 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9332 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9333 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9334 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9335 }
9336 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9337 {
9338 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9339 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9340 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9341 || pCtx->fs.Attr.n.u4Type > 11
9342 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9343 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9344 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9345 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9346 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9347 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9348 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9349 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9350 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9351 }
9352 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9353 {
9354 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9355 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9356 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9357 || pCtx->gs.Attr.n.u4Type > 11
9358 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9359 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9360 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9361 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9362 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9363 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9364 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9365 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9366 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9367 }
9368 /* 64-bit capable CPUs. */
9369#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9370 if (HMVMX_IS_64BIT_HOST_MODE())
9371 {
9372 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9373 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9374 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9375 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9376 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9377 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9378 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9379 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9380 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9381 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9382 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9383 }
9384#endif
9385 }
9386 else
9387 {
9388 /* V86 mode checks. */
9389 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9390 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9391 {
9392 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9393 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9394 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9395 }
9396 else
9397 {
9398 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9399 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9400 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9401 }
9402
9403 /* CS */
9404 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9405 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9406 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9407 /* SS */
9408 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9409 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9410 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9411 /* DS */
9412 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9413 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9414 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9415 /* ES */
9416 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9417 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9418 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9419 /* FS */
9420 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9421 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9422 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9423 /* GS */
9424 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9425 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9426 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9427 /* 64-bit capable CPUs. */
9428#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9429 if (HMVMX_IS_64BIT_HOST_MODE())
9430 {
9431 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9432 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9433 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9434 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9435 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9436 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9437 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9438 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9439 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9440 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9441 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9442 }
9443#endif
9444 }
9445
9446 /*
9447 * TR.
9448 */
9449 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9450 /* 64-bit capable CPUs. */
9451#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9452 if (HMVMX_IS_64BIT_HOST_MODE())
9453 {
9454 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9455 }
9456#endif
9457 if (fLongModeGuest)
9458 {
9459 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9460 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9461 }
9462 else
9463 {
9464 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9465 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9466 VMX_IGS_TR_ATTR_TYPE_INVALID);
9467 }
9468 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9469 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9470 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9471 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9472 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9473 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9474 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9475 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9476
9477 /*
9478 * GDTR and IDTR.
9479 */
9480#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9481 if (HMVMX_IS_64BIT_HOST_MODE())
9482 {
9483 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9484 AssertRCBreak(rc);
9485 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9486
9487 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9488 AssertRCBreak(rc);
9489 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9490 }
9491#endif
9492
9493 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9494 AssertRCBreak(rc);
9495 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9496
9497 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9498 AssertRCBreak(rc);
9499 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9500
9501 /*
9502 * Guest Non-Register State.
9503 */
9504 /* Activity State. */
9505 uint32_t u32ActivityState;
9506 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9507 AssertRCBreak(rc);
9508 HMVMX_CHECK_BREAK( !u32ActivityState
9509 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9510 VMX_IGS_ACTIVITY_STATE_INVALID);
9511 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9512 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9513 uint32_t u32IntrState;
9514 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9515 AssertRCBreak(rc);
9516 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9517 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9518 {
9519 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9520 }
9521
9522 /** @todo Activity state and injecting interrupts. Left as a todo since we
9523 * currently don't use activity states but ACTIVE. */
9524
9525 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9526 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9527
9528 /* Guest interruptibility-state. */
9529 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9530 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9531 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9532 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9533 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9534 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9535 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9536 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9537 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9538 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9539 {
9540 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9541 {
9542 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9543 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9544 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9545 }
9546 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9547 {
9548 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9549 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9550 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9551 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9552 }
9553 }
9554 /** @todo Assumes the processor is not in SMM. */
9555 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9556 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9557 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9558 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9559 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9560 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9561 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9562 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9563 {
9564 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9565 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9566 }
9567
9568 /* Pending debug exceptions. */
9569 if (HMVMX_IS_64BIT_HOST_MODE())
9570 {
9571 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9572 AssertRCBreak(rc);
9573 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9574 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9575 u32Val = u64Val; /* For pending debug exceptions checks below. */
9576 }
9577 else
9578 {
9579 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9580 AssertRCBreak(rc);
9581 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9582 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9583 }
9584
9585 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9586 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9587 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9588 {
9589 if ( (u32Eflags & X86_EFL_TF)
9590 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9591 {
9592 /* Bit 14 is PendingDebug.BS. */
9593 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9594 }
9595 if ( !(u32Eflags & X86_EFL_TF)
9596 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9597 {
9598 /* Bit 14 is PendingDebug.BS. */
9599 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9600 }
9601 }
9602
9603 /* VMCS link pointer. */
9604 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9605 AssertRCBreak(rc);
9606 if (u64Val != UINT64_C(0xffffffffffffffff))
9607 {
9608 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9609 /** @todo Bits beyond the processor's physical-address width MBZ. */
9610 /** @todo 32-bit located in memory referenced by value of this field (as a
9611 * physical address) must contain the processor's VMCS revision ID. */
9612 /** @todo SMM checks. */
9613 }
9614
9615 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9616 * not using Nested Paging? */
9617 if ( pVM->hm.s.fNestedPaging
9618 && !fLongModeGuest
9619 && CPUMIsGuestInPAEModeEx(pCtx))
9620 {
9621 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9622 AssertRCBreak(rc);
9623 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9624
9625 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9626 AssertRCBreak(rc);
9627 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9628
9629 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9630 AssertRCBreak(rc);
9631 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9632
9633 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9634 AssertRCBreak(rc);
9635 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9636 }
9637
9638 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9639 if (uError == VMX_IGS_ERROR)
9640 uError = VMX_IGS_REASON_NOT_FOUND;
9641 } while (0);
9642
9643 pVCpu->hm.s.u32HMError = uError;
9644 return uError;
9645
9646#undef HMVMX_ERROR_BREAK
9647#undef HMVMX_CHECK_BREAK
9648}
9649
9650/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9651/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9652/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9653
9654/** @name VM-exit handlers.
9655 * @{
9656 */
9657
9658/**
9659 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9660 */
9661HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9662{
9663 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9665 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9666 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9667 return VINF_SUCCESS;
9668 return VINF_EM_RAW_INTERRUPT;
9669}
9670
9671
9672/**
9673 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9674 */
9675HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9676{
9677 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9678 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9679
9680 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9681 AssertRCReturn(rc, rc);
9682
9683 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9684 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9685 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9686 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9687
9688 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9689 {
9690 /*
9691 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9692 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9693 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9694 *
9695 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9696 */
9697 VMXDispatchHostNmi();
9698 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9699 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9700 return VINF_SUCCESS;
9701 }
9702
9703 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9704 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9705 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9706 {
9707 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9708 return VINF_SUCCESS;
9709 }
9710 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9711 {
9712 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9713 return rc;
9714 }
9715
9716 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9717 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9718 switch (uIntType)
9719 {
9720 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9721 Assert(uVector == X86_XCPT_DB);
9722 /* no break */
9723 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9724 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9725 /* no break */
9726 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9727 {
9728 switch (uVector)
9729 {
9730 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9731 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9732 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9733 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9734 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9735 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9737 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9738 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9739 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9740 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9741 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9742 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9743 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9744 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9745 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9746 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9747 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9748 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9749#endif
9750 default:
9751 {
9752 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9753 AssertRCReturn(rc, rc);
9754
9755 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9756 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9757 {
9758 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9759 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9760 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9761
9762 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9763 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9764 AssertRCReturn(rc, rc);
9765 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9766 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9767 0 /* GCPtrFaultAddress */);
9768 AssertRCReturn(rc, rc);
9769 }
9770 else
9771 {
9772 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9773 pVCpu->hm.s.u32HMError = uVector;
9774 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9775 }
9776 break;
9777 }
9778 }
9779 break;
9780 }
9781
9782 default:
9783 {
9784 pVCpu->hm.s.u32HMError = uExitIntInfo;
9785 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9786 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9787 break;
9788 }
9789 }
9790 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9791 return rc;
9792}
9793
9794
9795/**
9796 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9797 */
9798HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9799{
9800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9801
9802 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9803 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9804
9805 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9806 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9807 return VINF_SUCCESS;
9808}
9809
9810
9811/**
9812 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9813 */
9814HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9815{
9816 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9817 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9818 HMVMX_RETURN_UNEXPECTED_EXIT();
9819}
9820
9821
9822/**
9823 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9824 */
9825HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9826{
9827 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9828 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9829 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9830}
9831
9832
9833/**
9834 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9835 */
9836HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9837{
9838 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9839 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9840 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9841}
9842
9843
9844/**
9845 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9846 */
9847HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9848{
9849 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9850 PVM pVM = pVCpu->CTX_SUFF(pVM);
9851 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9852 if (RT_LIKELY(rc == VINF_SUCCESS))
9853 {
9854 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9855 Assert(pVmxTransient->cbInstr == 2);
9856 }
9857 else
9858 {
9859 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9860 rc = VERR_EM_INTERPRETER;
9861 }
9862 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9863 return rc;
9864}
9865
9866
9867/**
9868 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9869 */
9870HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9871{
9872 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9873 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9874 AssertRCReturn(rc, rc);
9875
9876 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9877 return VINF_EM_RAW_EMULATE_INSTR;
9878
9879 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9880 HMVMX_RETURN_UNEXPECTED_EXIT();
9881}
9882
9883
9884/**
9885 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9886 */
9887HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9888{
9889 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9890 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9891 AssertRCReturn(rc, rc);
9892
9893 PVM pVM = pVCpu->CTX_SUFF(pVM);
9894 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9895 if (RT_LIKELY(rc == VINF_SUCCESS))
9896 {
9897 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9898 Assert(pVmxTransient->cbInstr == 2);
9899 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9900 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9901 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9902 }
9903 else
9904 {
9905 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9906 rc = VERR_EM_INTERPRETER;
9907 }
9908 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9909 return rc;
9910}
9911
9912
9913/**
9914 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9915 */
9916HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9917{
9918 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9919 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9920 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9921 AssertRCReturn(rc, rc);
9922
9923 PVM pVM = pVCpu->CTX_SUFF(pVM);
9924 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9925 if (RT_LIKELY(rc == VINF_SUCCESS))
9926 {
9927 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9928 Assert(pVmxTransient->cbInstr == 3);
9929 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9930 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9931 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9932 }
9933 else
9934 {
9935 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9936 rc = VERR_EM_INTERPRETER;
9937 }
9938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9939 return rc;
9940}
9941
9942
9943/**
9944 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9945 */
9946HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9947{
9948 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9949 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9950 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9951 AssertRCReturn(rc, rc);
9952
9953 PVM pVM = pVCpu->CTX_SUFF(pVM);
9954 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9955 if (RT_LIKELY(rc == VINF_SUCCESS))
9956 {
9957 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9958 Assert(pVmxTransient->cbInstr == 2);
9959 }
9960 else
9961 {
9962 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9963 rc = VERR_EM_INTERPRETER;
9964 }
9965 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9966 return rc;
9967}
9968
9969
9970/**
9971 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9972 */
9973HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9974{
9975 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9976 PVM pVM = pVCpu->CTX_SUFF(pVM);
9977 Assert(!pVM->hm.s.fNestedPaging);
9978
9979 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9980 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9981 AssertRCReturn(rc, rc);
9982
9983 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9984 rc = VBOXSTRICTRC_VAL(rc2);
9985 if (RT_LIKELY(rc == VINF_SUCCESS))
9986 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9987 else
9988 {
9989 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9990 pVmxTransient->uExitQualification, rc));
9991 }
9992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9993 return rc;
9994}
9995
9996
9997/**
9998 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9999 */
10000HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10001{
10002 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10003 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10004 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10005 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10006 AssertRCReturn(rc, rc);
10007
10008 PVM pVM = pVCpu->CTX_SUFF(pVM);
10009 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10010 if (RT_LIKELY(rc == VINF_SUCCESS))
10011 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10012 else
10013 {
10014 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10015 rc = VERR_EM_INTERPRETER;
10016 }
10017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10018 return rc;
10019}
10020
10021
10022/**
10023 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10024 */
10025HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10026{
10027 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10028 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10029 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10030 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10031 AssertRCReturn(rc, rc);
10032
10033 PVM pVM = pVCpu->CTX_SUFF(pVM);
10034 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10035 rc = VBOXSTRICTRC_VAL(rc2);
10036 if (RT_LIKELY( rc == VINF_SUCCESS
10037 || rc == VINF_EM_HALT))
10038 {
10039 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10040 AssertRCReturn(rc3, rc3);
10041
10042 if ( rc == VINF_EM_HALT
10043 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10044 {
10045 rc = VINF_SUCCESS;
10046 }
10047 }
10048 else
10049 {
10050 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10051 rc = VERR_EM_INTERPRETER;
10052 }
10053 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10054 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10055 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10056 return rc;
10057}
10058
10059
10060/**
10061 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10062 */
10063HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10064{
10065 /*
10066 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10067 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10068 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10069 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10070 */
10071 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10072 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10073 HMVMX_RETURN_UNEXPECTED_EXIT();
10074}
10075
10076
10077/**
10078 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10079 */
10080HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10081{
10082 /*
10083 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10084 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10085 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10086 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10087 */
10088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10089 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10090 HMVMX_RETURN_UNEXPECTED_EXIT();
10091}
10092
10093
10094/**
10095 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10096 */
10097HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10098{
10099 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10100 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10101 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10102 HMVMX_RETURN_UNEXPECTED_EXIT();
10103}
10104
10105
10106/**
10107 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10108 */
10109HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10110{
10111 /*
10112 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10113 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10114 * See Intel spec. 25.3 "Other Causes of VM-exits".
10115 */
10116 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10117 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10118 HMVMX_RETURN_UNEXPECTED_EXIT();
10119}
10120
10121
10122/**
10123 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10124 * VM-exit.
10125 */
10126HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10127{
10128 /*
10129 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10130 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10131 *
10132 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10133 * See Intel spec. "23.8 Restrictions on VMX operation".
10134 */
10135 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10136 return VINF_SUCCESS;
10137}
10138
10139
10140/**
10141 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10142 * VM-exit.
10143 */
10144HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10145{
10146 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10147 return VINF_EM_RESET;
10148}
10149
10150
10151/**
10152 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10153 */
10154HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10155{
10156 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10157 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10158 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10159 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10160 AssertRCReturn(rc, rc);
10161
10162 pMixedCtx->rip++;
10163 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10164 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10165 rc = VINF_SUCCESS;
10166 else
10167 rc = VINF_EM_HALT;
10168
10169 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10170 return rc;
10171}
10172
10173
10174/**
10175 * VM-exit handler for instructions that result in a #UD exception delivered to
10176 * the guest.
10177 */
10178HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10179{
10180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10181 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10182 return VINF_SUCCESS;
10183}
10184
10185
10186/**
10187 * VM-exit handler for expiry of the VMX preemption timer.
10188 */
10189HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10190{
10191 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10192
10193 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10194 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10195
10196 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10197 PVM pVM = pVCpu->CTX_SUFF(pVM);
10198 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10199 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10200 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10201}
10202
10203
10204/**
10205 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10206 */
10207HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10208{
10209 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10210
10211 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10212 /** @todo check if XSETBV is supported by the recompiler. */
10213 return VERR_EM_INTERPRETER;
10214}
10215
10216
10217/**
10218 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10219 */
10220HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10221{
10222 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10223
10224 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10225 /** @todo implement EMInterpretInvpcid() */
10226 return VERR_EM_INTERPRETER;
10227}
10228
10229
10230/**
10231 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10232 * Error VM-exit.
10233 */
10234HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10235{
10236 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10237 AssertRCReturn(rc, rc);
10238
10239 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10240 AssertRCReturn(rc, rc);
10241
10242 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10243 NOREF(uInvalidReason);
10244
10245#ifdef VBOX_STRICT
10246 uint32_t uIntrState;
10247 HMVMXHCUINTREG uHCReg;
10248 uint64_t u64Val;
10249 uint32_t u32Val;
10250
10251 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10252 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10253 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10254 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10255 AssertRCReturn(rc, rc);
10256
10257 Log4(("uInvalidReason %u\n", uInvalidReason));
10258 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10259 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10260 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10261 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10262
10263 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10264 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10265 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10266 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10267 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10268 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10269 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10270 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10271 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10272 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10273 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10274 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10275#else
10276 NOREF(pVmxTransient);
10277#endif
10278
10279 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10280 return VERR_VMX_INVALID_GUEST_STATE;
10281}
10282
10283
10284/**
10285 * VM-exit handler for VM-entry failure due to an MSR-load
10286 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10287 */
10288HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10289{
10290 NOREF(pVmxTransient);
10291 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10292 HMVMX_RETURN_UNEXPECTED_EXIT();
10293}
10294
10295
10296/**
10297 * VM-exit handler for VM-entry failure due to a machine-check event
10298 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10299 */
10300HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10301{
10302 NOREF(pVmxTransient);
10303 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10304 HMVMX_RETURN_UNEXPECTED_EXIT();
10305}
10306
10307
10308/**
10309 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10310 * theory.
10311 */
10312HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10313{
10314 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10315 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10316 return VERR_VMX_UNDEFINED_EXIT_CODE;
10317}
10318
10319
10320/**
10321 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10322 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10323 * Conditional VM-exit.
10324 */
10325HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10326{
10327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10328
10329 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10331 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10332 return VERR_EM_INTERPRETER;
10333 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10334 HMVMX_RETURN_UNEXPECTED_EXIT();
10335}
10336
10337
10338/**
10339 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10340 */
10341HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10342{
10343 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10344
10345 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10346 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10347 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10348 return VERR_EM_INTERPRETER;
10349 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10350 HMVMX_RETURN_UNEXPECTED_EXIT();
10351}
10352
10353
10354/**
10355 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10356 */
10357HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10358{
10359 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10360
10361 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10362 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10363 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10364 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10365 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10366 {
10367 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10368 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10369 }
10370 AssertRCReturn(rc, rc);
10371 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10372
10373#ifdef VBOX_STRICT
10374 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10375 {
10376 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10377 && pMixedCtx->ecx != MSR_K6_EFER)
10378 {
10379 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10380 HMVMX_RETURN_UNEXPECTED_EXIT();
10381 }
10382# if HC_ARCH_BITS == 64
10383 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10384 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10385 {
10386 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10387 HMVMX_RETURN_UNEXPECTED_EXIT();
10388 }
10389# endif
10390 }
10391#endif
10392
10393 PVM pVM = pVCpu->CTX_SUFF(pVM);
10394 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10395 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10396 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10397 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10398 if (RT_LIKELY(rc == VINF_SUCCESS))
10399 {
10400 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10401 Assert(pVmxTransient->cbInstr == 2);
10402 }
10403 return rc;
10404}
10405
10406
10407/**
10408 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10409 */
10410HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10411{
10412 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10413 PVM pVM = pVCpu->CTX_SUFF(pVM);
10414 int rc = VINF_SUCCESS;
10415
10416 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10417 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10418 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10419 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10420 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10421 {
10422 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10423 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10424 }
10425 AssertRCReturn(rc, rc);
10426 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10427
10428 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10429 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10431
10432 if (RT_LIKELY(rc == VINF_SUCCESS))
10433 {
10434 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10435
10436 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10437 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10438 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10439 {
10440 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10441 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10442 EMInterpretWrmsr() changes it. */
10443 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10444 }
10445 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10446 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10447 else if (pMixedCtx->ecx == MSR_K6_EFER)
10448 {
10449 /*
10450 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10451 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10452 * the other bits as well, SCE and NXE. See @bugref{7368}.
10453 */
10454 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10455 }
10456
10457 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10458 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10459 {
10460 switch (pMixedCtx->ecx)
10461 {
10462 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10463 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10464 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10465 case MSR_K8_FS_BASE: /* no break */
10466 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10467 case MSR_K6_EFER: /* already handled above */ break;
10468 default:
10469 {
10470 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10471 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10472#if HC_ARCH_BITS == 64
10473 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10474 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10475#endif
10476 break;
10477 }
10478 }
10479 }
10480#ifdef VBOX_STRICT
10481 else
10482 {
10483 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10484 switch (pMixedCtx->ecx)
10485 {
10486 case MSR_IA32_SYSENTER_CS:
10487 case MSR_IA32_SYSENTER_EIP:
10488 case MSR_IA32_SYSENTER_ESP:
10489 case MSR_K8_FS_BASE:
10490 case MSR_K8_GS_BASE:
10491 {
10492 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10493 HMVMX_RETURN_UNEXPECTED_EXIT();
10494 }
10495
10496 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10497 default:
10498 {
10499 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10500 {
10501 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10502 if (pMixedCtx->ecx != MSR_K6_EFER)
10503 {
10504 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10505 pMixedCtx->ecx));
10506 HMVMX_RETURN_UNEXPECTED_EXIT();
10507 }
10508 }
10509
10510#if HC_ARCH_BITS == 64
10511 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10512 {
10513 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10514 HMVMX_RETURN_UNEXPECTED_EXIT();
10515 }
10516#endif
10517 break;
10518 }
10519 }
10520 }
10521#endif /* VBOX_STRICT */
10522 }
10523 return rc;
10524}
10525
10526
10527/**
10528 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10529 */
10530HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10531{
10532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10533
10534 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10536 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10537 return VERR_EM_INTERPRETER;
10538 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10539 HMVMX_RETURN_UNEXPECTED_EXIT();
10540}
10541
10542
10543/**
10544 * VM-exit handler for when the TPR value is lowered below the specified
10545 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10546 */
10547HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10548{
10549 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10550 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10551
10552 /*
10553 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10554 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10555 * resume guest execution.
10556 */
10557 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10558 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10559 return VINF_SUCCESS;
10560}
10561
10562
10563/**
10564 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10565 * VM-exit.
10566 *
10567 * @retval VINF_SUCCESS when guest execution can continue.
10568 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10569 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10570 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10571 * recompiler.
10572 */
10573HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10574{
10575 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10576 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10577 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10578 AssertRCReturn(rc, rc);
10579
10580 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10581 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10582 PVM pVM = pVCpu->CTX_SUFF(pVM);
10583 switch (uAccessType)
10584 {
10585 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10586 {
10587#if 0
10588 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10589 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10590#else
10591 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10592 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10593 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10594#endif
10595 AssertRCReturn(rc, rc);
10596
10597 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10598 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10599 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10600 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10601
10602 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10603 {
10604 case 0: /* CR0 */
10605 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10606 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10607 break;
10608 case 2: /* CR2 */
10609 /* Nothing to do here, CR2 it's not part of the VMCS. */
10610 break;
10611 case 3: /* CR3 */
10612 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10613 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10614 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10615 break;
10616 case 4: /* CR4 */
10617 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10618 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10619 break;
10620 case 8: /* CR8 */
10621 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10622 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10623 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10624 break;
10625 default:
10626 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10627 break;
10628 }
10629
10630 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10631 break;
10632 }
10633
10634 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10635 {
10636 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10637 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10638 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10639 AssertRCReturn(rc, rc);
10640 Assert( !pVM->hm.s.fNestedPaging
10641 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10642 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10643
10644 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10645 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10646 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10647
10648 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10649 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10650 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10651 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10652 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10653 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10654 break;
10655 }
10656
10657 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10658 {
10659 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10660 AssertRCReturn(rc, rc);
10661 rc = EMInterpretCLTS(pVM, pVCpu);
10662 AssertRCReturn(rc, rc);
10663 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10664 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10665 Log4(("CRX CLTS write rc=%d\n", rc));
10666 break;
10667 }
10668
10669 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10670 {
10671 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10672 AssertRCReturn(rc, rc);
10673 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10674 if (RT_LIKELY(rc == VINF_SUCCESS))
10675 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10677 Log4(("CRX LMSW write rc=%d\n", rc));
10678 break;
10679 }
10680
10681 default:
10682 {
10683 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10684 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10685 }
10686 }
10687
10688 /* Validate possible error codes. */
10689 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10690 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10691 if (RT_SUCCESS(rc))
10692 {
10693 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10694 AssertRCReturn(rc2, rc2);
10695 }
10696
10697 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10698 return rc;
10699}
10700
10701
10702/**
10703 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10704 * VM-exit.
10705 */
10706HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10707{
10708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10709 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10710
10711 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10712 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10713 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10714 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10715 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10716 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10717 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10718 AssertRCReturn(rc2, rc2);
10719
10720 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10721 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10722 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10723 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10724 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10725 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10726 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10727 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10728
10729 /* I/O operation lookup arrays. */
10730 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10731 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10732
10733 VBOXSTRICTRC rcStrict;
10734 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10735 const uint32_t cbInstr = pVmxTransient->cbInstr;
10736 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10737 PVM pVM = pVCpu->CTX_SUFF(pVM);
10738 if (fIOString)
10739 {
10740#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10741 /*
10742 * INS/OUTS - I/O String instruction.
10743 *
10744 * Use instruction-information if available, otherwise fall back on
10745 * interpreting the instruction.
10746 */
10747 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10748 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10749 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10750 {
10751 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10752 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10753 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10754 AssertRCReturn(rc2, rc2);
10755 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10756 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10757 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10758 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10759 if (fIOWrite)
10760 {
10761 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10762 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10763 }
10764 else
10765 {
10766 /*
10767 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10768 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10769 * See Intel Instruction spec. for "INS".
10770 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10771 */
10772 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10773 }
10774 }
10775 else
10776 {
10777 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10778 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10779 AssertRCReturn(rc2, rc2);
10780 rcStrict = IEMExecOne(pVCpu);
10781 }
10782 /** @todo IEM needs to be setting these flags somehow. */
10783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10784 fUpdateRipAlready = true;
10785#else
10786 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10787 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10788 if (RT_SUCCESS(rcStrict))
10789 {
10790 if (fIOWrite)
10791 {
10792 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10793 (DISCPUMODE)pDis->uAddrMode, cbValue);
10794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10795 }
10796 else
10797 {
10798 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10799 (DISCPUMODE)pDis->uAddrMode, cbValue);
10800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10801 }
10802 }
10803 else
10804 {
10805 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10806 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10807 }
10808#endif
10809 }
10810 else
10811 {
10812 /*
10813 * IN/OUT - I/O instruction.
10814 */
10815 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10816 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10817 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10818 if (fIOWrite)
10819 {
10820 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10821 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10822 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10824 }
10825 else
10826 {
10827 uint32_t u32Result = 0;
10828 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10829 if (IOM_SUCCESS(rcStrict))
10830 {
10831 /* Save result of I/O IN instr. in AL/AX/EAX. */
10832 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10833 }
10834 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10835 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10836 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10837 }
10838 }
10839
10840 if (IOM_SUCCESS(rcStrict))
10841 {
10842 if (!fUpdateRipAlready)
10843 {
10844 pMixedCtx->rip += cbInstr;
10845 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10846 }
10847
10848 /*
10849 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10850 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10851 */
10852 if (fIOString)
10853 {
10854 /** @todo Single-step for INS/OUTS with REP prefix? */
10855 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10856 }
10857 else if (fStepping)
10858 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10859
10860 /*
10861 * If any I/O breakpoints are armed, we need to check if one triggered
10862 * and take appropriate action.
10863 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10864 */
10865 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10866 AssertRCReturn(rc2, rc2);
10867
10868 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10869 * execution engines about whether hyper BPs and such are pending. */
10870 uint32_t const uDr7 = pMixedCtx->dr[7];
10871 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10872 && X86_DR7_ANY_RW_IO(uDr7)
10873 && (pMixedCtx->cr4 & X86_CR4_DE))
10874 || DBGFBpIsHwIoArmed(pVM)))
10875 {
10876 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10877
10878 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10879 VMMRZCallRing3Disable(pVCpu);
10880 HM_DISABLE_PREEMPT_IF_NEEDED();
10881
10882 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10883
10884 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10885 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10886 {
10887 /* Raise #DB. */
10888 if (fIsGuestDbgActive)
10889 ASMSetDR6(pMixedCtx->dr[6]);
10890 if (pMixedCtx->dr[7] != uDr7)
10891 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10892
10893 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10894 }
10895 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10896 else if ( rcStrict2 != VINF_SUCCESS
10897 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10898 rcStrict = rcStrict2;
10899
10900 HM_RESTORE_PREEMPT_IF_NEEDED();
10901 VMMRZCallRing3Enable(pVCpu);
10902 }
10903 }
10904
10905#ifdef DEBUG
10906 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10907 Assert(!fIOWrite);
10908 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10909 Assert(fIOWrite);
10910 else
10911 {
10912 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10913 * statuses, that the VMM device and some others may return. See
10914 * IOM_SUCCESS() for guidance. */
10915 AssertMsg( RT_FAILURE(rcStrict)
10916 || rcStrict == VINF_SUCCESS
10917 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10918 || rcStrict == VINF_EM_DBG_BREAKPOINT
10919 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10920 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10921 }
10922#endif
10923
10924 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10925 return VBOXSTRICTRC_TODO(rcStrict);
10926}
10927
10928
10929/**
10930 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10931 * VM-exit.
10932 */
10933HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10934{
10935 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10936
10937 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10938 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10939 AssertRCReturn(rc, rc);
10940 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10941 {
10942 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10943 AssertRCReturn(rc, rc);
10944 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10945 {
10946 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10947
10948 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10949 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10950
10951 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10952 Assert(!pVCpu->hm.s.Event.fPending);
10953 pVCpu->hm.s.Event.fPending = true;
10954 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10955 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10956 AssertRCReturn(rc, rc);
10957 if (fErrorCodeValid)
10958 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10959 else
10960 pVCpu->hm.s.Event.u32ErrCode = 0;
10961 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10962 && uVector == X86_XCPT_PF)
10963 {
10964 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10965 }
10966
10967 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10968 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10969 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10970 }
10971 }
10972
10973 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10974 * emulation. */
10975 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10976 return VERR_EM_INTERPRETER;
10977}
10978
10979
10980/**
10981 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10982 */
10983HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10984{
10985 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10986 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10987 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10988 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10989 AssertRCReturn(rc, rc);
10990 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10991 return VINF_EM_DBG_STEPPED;
10992}
10993
10994
10995/**
10996 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10997 */
10998HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10999{
11000 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11001
11002 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11003 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11004 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11005 return VINF_SUCCESS;
11006 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11007 return rc;
11008
11009#if 0
11010 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11011 * just sync the whole thing. */
11012 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11013#else
11014 /* Aggressive state sync. for now. */
11015 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11016 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11017 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11018#endif
11019 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11020 AssertRCReturn(rc, rc);
11021
11022 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11023 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11024 switch (uAccessType)
11025 {
11026 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11027 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11028 {
11029 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11030 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11031 {
11032 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11033 }
11034
11035 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11036 GCPhys &= PAGE_BASE_GC_MASK;
11037 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11038 PVM pVM = pVCpu->CTX_SUFF(pVM);
11039 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11040 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11041
11042 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11043 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11044 CPUMCTX2CORE(pMixedCtx), GCPhys);
11045 rc = VBOXSTRICTRC_VAL(rc2);
11046 Log4(("ApicAccess rc=%d\n", rc));
11047 if ( rc == VINF_SUCCESS
11048 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11049 || rc == VERR_PAGE_NOT_PRESENT)
11050 {
11051 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11052 | HM_CHANGED_GUEST_RSP
11053 | HM_CHANGED_GUEST_RFLAGS
11054 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11055 rc = VINF_SUCCESS;
11056 }
11057 break;
11058 }
11059
11060 default:
11061 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11062 rc = VINF_EM_RAW_EMULATE_INSTR;
11063 break;
11064 }
11065
11066 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11067 return rc;
11068}
11069
11070
11071/**
11072 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11073 * VM-exit.
11074 */
11075HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11076{
11077 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11078
11079 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11080 if (pVmxTransient->fWasGuestDebugStateActive)
11081 {
11082 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11083 HMVMX_RETURN_UNEXPECTED_EXIT();
11084 }
11085
11086 int rc = VERR_INTERNAL_ERROR_5;
11087 if ( !DBGFIsStepping(pVCpu)
11088 && !pVCpu->hm.s.fSingleInstruction
11089 && !pVmxTransient->fWasHyperDebugStateActive)
11090 {
11091 /* Don't intercept MOV DRx and #DB any more. */
11092 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11093 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11094 AssertRCReturn(rc, rc);
11095
11096 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11097 {
11098#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11099 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11100 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11101 AssertRCReturn(rc, rc);
11102#endif
11103 }
11104
11105 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11106 VMMRZCallRing3Disable(pVCpu);
11107 HM_DISABLE_PREEMPT_IF_NEEDED();
11108
11109 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11110 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11111 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11112
11113 HM_RESTORE_PREEMPT_IF_NEEDED();
11114 VMMRZCallRing3Enable(pVCpu);
11115
11116#ifdef VBOX_WITH_STATISTICS
11117 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11118 AssertRCReturn(rc, rc);
11119 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11121 else
11122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11123#endif
11124 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11125 return VINF_SUCCESS;
11126 }
11127
11128 /*
11129 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11130 * Update the segment registers and DR7 from the CPU.
11131 */
11132 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11133 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11134 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11135 AssertRCReturn(rc, rc);
11136 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11137
11138 PVM pVM = pVCpu->CTX_SUFF(pVM);
11139 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11140 {
11141 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11142 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11143 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11144 if (RT_SUCCESS(rc))
11145 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11147 }
11148 else
11149 {
11150 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11151 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11152 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11154 }
11155
11156 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11157 if (RT_SUCCESS(rc))
11158 {
11159 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11160 AssertRCReturn(rc2, rc2);
11161 }
11162 return rc;
11163}
11164
11165
11166/**
11167 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11168 * Conditional VM-exit.
11169 */
11170HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11171{
11172 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11173 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11174
11175 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11176 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11177 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11178 return VINF_SUCCESS;
11179 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11180 return rc;
11181
11182 RTGCPHYS GCPhys = 0;
11183 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11184
11185#if 0
11186 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11187#else
11188 /* Aggressive state sync. for now. */
11189 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11190 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11191 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11192#endif
11193 AssertRCReturn(rc, rc);
11194
11195 /*
11196 * If we succeed, resume guest execution.
11197 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11198 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11199 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11200 * weird case. See @bugref{6043}.
11201 */
11202 PVM pVM = pVCpu->CTX_SUFF(pVM);
11203 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11204 rc = VBOXSTRICTRC_VAL(rc2);
11205 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11206 if ( rc == VINF_SUCCESS
11207 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11208 || rc == VERR_PAGE_NOT_PRESENT)
11209 {
11210 /* Successfully handled MMIO operation. */
11211 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11212 | HM_CHANGED_GUEST_RSP
11213 | HM_CHANGED_GUEST_RFLAGS
11214 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11215 rc = VINF_SUCCESS;
11216 }
11217 return rc;
11218}
11219
11220
11221/**
11222 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11223 * VM-exit.
11224 */
11225HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11226{
11227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11228 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11229
11230 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11231 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11232 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11233 return VINF_SUCCESS;
11234 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11235 return rc;
11236
11237 RTGCPHYS GCPhys = 0;
11238 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11239 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11240#if 0
11241 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11242#else
11243 /* Aggressive state sync. for now. */
11244 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11245 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11246 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11247#endif
11248 AssertRCReturn(rc, rc);
11249
11250 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11251 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11252
11253 RTGCUINT uErrorCode = 0;
11254 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11255 uErrorCode |= X86_TRAP_PF_ID;
11256 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11257 uErrorCode |= X86_TRAP_PF_RW;
11258 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11259 uErrorCode |= X86_TRAP_PF_P;
11260
11261 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11262
11263 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11264 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11265
11266 /* Handle the pagefault trap for the nested shadow table. */
11267 PVM pVM = pVCpu->CTX_SUFF(pVM);
11268 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11269 TRPMResetTrap(pVCpu);
11270
11271 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11272 if ( rc == VINF_SUCCESS
11273 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11274 || rc == VERR_PAGE_NOT_PRESENT)
11275 {
11276 /* Successfully synced our nested page tables. */
11277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11278 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11279 | HM_CHANGED_GUEST_RSP
11280 | HM_CHANGED_GUEST_RFLAGS);
11281 return VINF_SUCCESS;
11282 }
11283
11284 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11285 return rc;
11286}
11287
11288/** @} */
11289
11290/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11291/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11292/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11293
11294/** @name VM-exit exception handlers.
11295 * @{
11296 */
11297
11298/**
11299 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11300 */
11301static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11302{
11303 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11305
11306 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11307 AssertRCReturn(rc, rc);
11308
11309 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11310 {
11311 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11312 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11313
11314 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11315 * provides VM-exit instruction length. If this causes problem later,
11316 * disassemble the instruction like it's done on AMD-V. */
11317 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11318 AssertRCReturn(rc2, rc2);
11319 return rc;
11320 }
11321
11322 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11323 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11324 return rc;
11325}
11326
11327
11328/**
11329 * VM-exit exception handler for #BP (Breakpoint exception).
11330 */
11331static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11332{
11333 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11335
11336 /** @todo Try optimize this by not saving the entire guest state unless
11337 * really needed. */
11338 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11339 AssertRCReturn(rc, rc);
11340
11341 PVM pVM = pVCpu->CTX_SUFF(pVM);
11342 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11343 if (rc == VINF_EM_RAW_GUEST_TRAP)
11344 {
11345 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11346 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11347 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11348 AssertRCReturn(rc, rc);
11349
11350 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11351 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11352 }
11353
11354 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11355 return rc;
11356}
11357
11358
11359/**
11360 * VM-exit exception handler for #DB (Debug exception).
11361 */
11362static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11363{
11364 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11366 Log6(("XcptDB\n"));
11367
11368 /*
11369 * Get the DR6-like values from the exit qualification and pass it to DBGF
11370 * for processing.
11371 */
11372 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11373 AssertRCReturn(rc, rc);
11374
11375 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11376 uint64_t uDR6 = X86_DR6_INIT_VAL;
11377 uDR6 |= ( pVmxTransient->uExitQualification
11378 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11379
11380 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11381 if (rc == VINF_EM_RAW_GUEST_TRAP)
11382 {
11383 /*
11384 * The exception was for the guest. Update DR6, DR7.GD and
11385 * IA32_DEBUGCTL.LBR before forwarding it.
11386 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11387 */
11388 VMMRZCallRing3Disable(pVCpu);
11389 HM_DISABLE_PREEMPT_IF_NEEDED();
11390
11391 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11392 pMixedCtx->dr[6] |= uDR6;
11393 if (CPUMIsGuestDebugStateActive(pVCpu))
11394 ASMSetDR6(pMixedCtx->dr[6]);
11395
11396 HM_RESTORE_PREEMPT_IF_NEEDED();
11397 VMMRZCallRing3Enable(pVCpu);
11398
11399 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11400 AssertRCReturn(rc, rc);
11401
11402 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11403 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11404
11405 /* Paranoia. */
11406 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11407 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11408
11409 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11410 AssertRCReturn(rc, rc);
11411
11412 /*
11413 * Raise #DB in the guest.
11414 *
11415 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11416 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11417 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11418 *
11419 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11420 */
11421 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11422 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11423 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11424 AssertRCReturn(rc, rc);
11425 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11426 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11427 return VINF_SUCCESS;
11428 }
11429
11430 /*
11431 * Not a guest trap, must be a hypervisor related debug event then.
11432 * Update DR6 in case someone is interested in it.
11433 */
11434 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11435 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11436 CPUMSetHyperDR6(pVCpu, uDR6);
11437
11438 return rc;
11439}
11440
11441
11442/**
11443 * VM-exit exception handler for #NM (Device-not-available exception: floating
11444 * point exception).
11445 */
11446static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11447{
11448 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11449
11450 /* We require CR0 and EFER. EFER is always up-to-date. */
11451 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11452 AssertRCReturn(rc, rc);
11453
11454 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11455 VMMRZCallRing3Disable(pVCpu);
11456 HM_DISABLE_PREEMPT_IF_NEEDED();
11457
11458 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11459 if (pVmxTransient->fWasGuestFPUStateActive)
11460 {
11461 rc = VINF_EM_RAW_GUEST_TRAP;
11462 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11463 }
11464 else
11465 {
11466#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11467 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11468#endif
11469 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11470 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11471 }
11472
11473 HM_RESTORE_PREEMPT_IF_NEEDED();
11474 VMMRZCallRing3Enable(pVCpu);
11475
11476 if (rc == VINF_SUCCESS)
11477 {
11478 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11479 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11481 pVCpu->hm.s.fUseGuestFpu = true;
11482 }
11483 else
11484 {
11485 /* Forward #NM to the guest. */
11486 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11487 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11488 AssertRCReturn(rc, rc);
11489 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11490 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11491 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11492 }
11493
11494 return VINF_SUCCESS;
11495}
11496
11497
11498/**
11499 * VM-exit exception handler for #GP (General-protection exception).
11500 *
11501 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11502 */
11503static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11504{
11505 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11507
11508 int rc = VERR_INTERNAL_ERROR_5;
11509 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11510 {
11511#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11512 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11513 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11514 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11515 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11516 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11517 AssertRCReturn(rc, rc);
11518 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11519 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11520 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11521 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11522 return rc;
11523#else
11524 /* We don't intercept #GP. */
11525 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11526 NOREF(pVmxTransient);
11527 return VERR_VMX_UNEXPECTED_EXCEPTION;
11528#endif
11529 }
11530
11531 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11532 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11533
11534 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11535 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11536 AssertRCReturn(rc, rc);
11537
11538 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11539 uint32_t cbOp = 0;
11540 PVM pVM = pVCpu->CTX_SUFF(pVM);
11541 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11542 if (RT_SUCCESS(rc))
11543 {
11544 rc = VINF_SUCCESS;
11545 Assert(cbOp == pDis->cbInstr);
11546 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11547 switch (pDis->pCurInstr->uOpcode)
11548 {
11549 case OP_CLI:
11550 {
11551 pMixedCtx->eflags.Bits.u1IF = 0;
11552 pMixedCtx->rip += pDis->cbInstr;
11553 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11554 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11556 break;
11557 }
11558
11559 case OP_STI:
11560 {
11561 pMixedCtx->eflags.Bits.u1IF = 1;
11562 pMixedCtx->rip += pDis->cbInstr;
11563 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11564 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11565 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11566 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11567 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11568 break;
11569 }
11570
11571 case OP_HLT:
11572 {
11573 rc = VINF_EM_HALT;
11574 pMixedCtx->rip += pDis->cbInstr;
11575 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11576 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11577 break;
11578 }
11579
11580 case OP_POPF:
11581 {
11582 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11583 uint32_t cbParm;
11584 uint32_t uMask;
11585 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11586 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11587 {
11588 cbParm = 4;
11589 uMask = 0xffffffff;
11590 }
11591 else
11592 {
11593 cbParm = 2;
11594 uMask = 0xffff;
11595 }
11596
11597 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11598 RTGCPTR GCPtrStack = 0;
11599 X86EFLAGS Eflags;
11600 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11601 &GCPtrStack);
11602 if (RT_SUCCESS(rc))
11603 {
11604 Assert(sizeof(Eflags.u32) >= cbParm);
11605 Eflags.u32 = 0;
11606 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11607 }
11608 if (RT_FAILURE(rc))
11609 {
11610 rc = VERR_EM_INTERPRETER;
11611 break;
11612 }
11613 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11614 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11615 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11616 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11617 pMixedCtx->esp += cbParm;
11618 pMixedCtx->esp &= uMask;
11619 pMixedCtx->rip += pDis->cbInstr;
11620 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11621 | HM_CHANGED_GUEST_RSP
11622 | HM_CHANGED_GUEST_RFLAGS);
11623 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11624 if (fStepping)
11625 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11626
11627 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11628 break;
11629 }
11630
11631 case OP_PUSHF:
11632 {
11633 uint32_t cbParm;
11634 uint32_t uMask;
11635 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11636 {
11637 cbParm = 4;
11638 uMask = 0xffffffff;
11639 }
11640 else
11641 {
11642 cbParm = 2;
11643 uMask = 0xffff;
11644 }
11645
11646 /* Get the stack pointer & push the contents of eflags onto the stack. */
11647 RTGCPTR GCPtrStack = 0;
11648 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11649 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11650 if (RT_FAILURE(rc))
11651 {
11652 rc = VERR_EM_INTERPRETER;
11653 break;
11654 }
11655 X86EFLAGS Eflags = pMixedCtx->eflags;
11656 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11657 Eflags.Bits.u1RF = 0;
11658 Eflags.Bits.u1VM = 0;
11659
11660 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11661 if (RT_FAILURE(rc))
11662 {
11663 rc = VERR_EM_INTERPRETER;
11664 break;
11665 }
11666 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11667 pMixedCtx->esp -= cbParm;
11668 pMixedCtx->esp &= uMask;
11669 pMixedCtx->rip += pDis->cbInstr;
11670 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11671 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11673 break;
11674 }
11675
11676 case OP_IRET:
11677 {
11678 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11679 * instruction reference. */
11680 RTGCPTR GCPtrStack = 0;
11681 uint32_t uMask = 0xffff;
11682 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11683 uint16_t aIretFrame[3];
11684 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11685 {
11686 rc = VERR_EM_INTERPRETER;
11687 break;
11688 }
11689 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11690 &GCPtrStack);
11691 if (RT_SUCCESS(rc))
11692 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11693 if (RT_FAILURE(rc))
11694 {
11695 rc = VERR_EM_INTERPRETER;
11696 break;
11697 }
11698 pMixedCtx->eip = 0;
11699 pMixedCtx->ip = aIretFrame[0];
11700 pMixedCtx->cs.Sel = aIretFrame[1];
11701 pMixedCtx->cs.ValidSel = aIretFrame[1];
11702 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11703 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11704 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11705 pMixedCtx->sp += sizeof(aIretFrame);
11706 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11707 | HM_CHANGED_GUEST_SEGMENT_REGS
11708 | HM_CHANGED_GUEST_RSP
11709 | HM_CHANGED_GUEST_RFLAGS);
11710 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11711 if (fStepping)
11712 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11713 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11715 break;
11716 }
11717
11718 case OP_INT:
11719 {
11720 uint16_t uVector = pDis->Param1.uValue & 0xff;
11721 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11722 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11723 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11724 break;
11725 }
11726
11727 case OP_INTO:
11728 {
11729 if (pMixedCtx->eflags.Bits.u1OF)
11730 {
11731 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11732 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11734 }
11735 break;
11736 }
11737
11738 default:
11739 {
11740 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11741 EMCODETYPE_SUPERVISOR);
11742 rc = VBOXSTRICTRC_VAL(rc2);
11743 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11744 /** @todo We have to set pending-debug exceptions here when the guest is
11745 * single-stepping depending on the instruction that was interpreted. */
11746 Log4(("#GP rc=%Rrc\n", rc));
11747 break;
11748 }
11749 }
11750 }
11751 else
11752 rc = VERR_EM_INTERPRETER;
11753
11754 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11755 ("#GP Unexpected rc=%Rrc\n", rc));
11756 return rc;
11757}
11758
11759
11760#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11761/**
11762 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11763 * the exception reported in the VMX transient structure back into the VM.
11764 *
11765 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11766 * up-to-date.
11767 */
11768static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11769{
11770 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11771
11772 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11773 hmR0VmxCheckExitDueToEventDelivery(). */
11774 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11775 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11776 AssertRCReturn(rc, rc);
11777 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11778
11779#ifdef DEBUG_ramshankar
11780 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11781 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11782 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11783#endif
11784
11785 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11786 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11787 return VINF_SUCCESS;
11788}
11789#endif
11790
11791
11792/**
11793 * VM-exit exception handler for #PF (Page-fault exception).
11794 */
11795static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11796{
11797 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11798 PVM pVM = pVCpu->CTX_SUFF(pVM);
11799 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11800 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11801 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11802 AssertRCReturn(rc, rc);
11803
11804#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11805 if (pVM->hm.s.fNestedPaging)
11806 {
11807 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11808 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11809 {
11810 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11811 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11812 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11813 }
11814 else
11815 {
11816 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11817 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11818 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11819 }
11820 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11821 return rc;
11822 }
11823#else
11824 Assert(!pVM->hm.s.fNestedPaging);
11825 NOREF(pVM);
11826#endif
11827
11828 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11829 AssertRCReturn(rc, rc);
11830
11831 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11832 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11833
11834 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11835 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11836 (RTGCPTR)pVmxTransient->uExitQualification);
11837
11838 Log4(("#PF: rc=%Rrc\n", rc));
11839 if (rc == VINF_SUCCESS)
11840 {
11841 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11842 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11843 * memory? We don't update the whole state here... */
11844 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11845 | HM_CHANGED_GUEST_RSP
11846 | HM_CHANGED_GUEST_RFLAGS
11847 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11848 TRPMResetTrap(pVCpu);
11849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11850 return rc;
11851 }
11852 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11853 {
11854 if (!pVmxTransient->fVectoringPF)
11855 {
11856 /* It's a guest page fault and needs to be reflected to the guest. */
11857 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11858 TRPMResetTrap(pVCpu);
11859 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11860 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11861 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11862 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11863 }
11864 else
11865 {
11866 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11867 TRPMResetTrap(pVCpu);
11868 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11869 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11870 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11871 }
11872
11873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11874 return VINF_SUCCESS;
11875 }
11876
11877 TRPMResetTrap(pVCpu);
11878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11879 return rc;
11880}
11881
11882/** @} */
11883
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette