VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 52066

Last change on this file since 52066 was 52066, checked in by vboxsync, 10 years ago

VMM: Fixed potential bug in AMD-V NMI injection when interrupt shadowing is in effect, renamed INHIBIT_NMIS to BLOCK_NMIS to match Intel specs.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 498.1 KB
Line 
1/* $Id: HMVMXR0.cpp 52066 2014-07-17 07:02:33Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2014 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#include <VBox/vmm/gim.h>
37#ifdef VBOX_WITH_REM
38# include <VBox/vmm/rem.h>
39#endif
40#ifdef DEBUG_ramshankar
41# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
42# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
43# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
44# define HMVMX_ALWAYS_CHECK_GUEST_STATE
45# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
46# define HMVMX_ALWAYS_TRAP_PF
47# define HMVMX_ALWAYS_SWAP_FPU_STATE
48# define HMVMX_ALWAYS_FLUSH_TLB
49# define HMVMX_ALWAYS_SWAP_EFER
50#endif
51
52
53/*******************************************************************************
54* Defined Constants And Macros *
55*******************************************************************************/
56#if defined(RT_ARCH_AMD64)
57# define HMVMX_IS_64BIT_HOST_MODE() (true)
58typedef RTHCUINTREG HMVMXHCUINTREG;
59#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
60extern "C" uint32_t g_fVMXIs64bitHost;
61# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
62typedef uint64_t HMVMXHCUINTREG;
63#else
64# define HMVMX_IS_64BIT_HOST_MODE() (false)
65typedef RTHCUINTREG HMVMXHCUINTREG;
66#endif
67
68/** Use the function table. */
69#define HMVMX_USE_FUNCTION_TABLE
70
71/** Determine which tagged-TLB flush handler to use. */
72#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
73#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
74#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
75#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
76
77/** @name Updated-guest-state flags.
78 * @{ */
79#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
80#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
81#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
82#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
83#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
84#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
85#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
86#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
87#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
88#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
89#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
90#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
91#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
92#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
93#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
94#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
95#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
96#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
97#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
98#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
99 | HMVMX_UPDATED_GUEST_RSP \
100 | HMVMX_UPDATED_GUEST_RFLAGS \
101 | HMVMX_UPDATED_GUEST_CR0 \
102 | HMVMX_UPDATED_GUEST_CR3 \
103 | HMVMX_UPDATED_GUEST_CR4 \
104 | HMVMX_UPDATED_GUEST_GDTR \
105 | HMVMX_UPDATED_GUEST_IDTR \
106 | HMVMX_UPDATED_GUEST_LDTR \
107 | HMVMX_UPDATED_GUEST_TR \
108 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
109 | HMVMX_UPDATED_GUEST_DEBUG \
110 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
111 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
112 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
113 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
114 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
115 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
116 | HMVMX_UPDATED_GUEST_APIC_STATE)
117/** @} */
118
119/** @name
120 * Flags to skip redundant reads of some common VMCS fields that are not part of
121 * the guest-CPU state but are in the transient structure.
122 */
123#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
124#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
127#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
128#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
129#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
130/** @} */
131
132/** @name
133 * States of the VMCS.
134 *
135 * This does not reflect all possible VMCS states but currently only those
136 * needed for maintaining the VMCS consistently even when thread-context hooks
137 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
138 */
139#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
140#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
141#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
142/** @} */
143
144/**
145 * Exception bitmap mask for real-mode guests (real-on-v86).
146 *
147 * We need to intercept all exceptions manually (except #PF). #NM is also
148 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
149 * even in real-mode if we have Nested Paging support.
150 */
151#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
152 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
153 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
154 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
155 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
156 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
157 | RT_BIT(X86_XCPT_XF))
158
159/**
160 * Exception bitmap mask for all contributory exceptions.
161 *
162 * Page fault is deliberately excluded here as it's conditional as to whether
163 * it's contributory or benign. Page faults are handled separately.
164 */
165#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
166 | RT_BIT(X86_XCPT_DE))
167
168/** Maximum VM-instruction error number. */
169#define HMVMX_INSTR_ERROR_MAX 28
170
171/** Profiling macro. */
172#ifdef HM_PROFILE_EXIT_DISPATCH
173# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
175#else
176# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
177# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
178#endif
179
180/** Assert that preemption is disabled or covered by thread-context hooks. */
181#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
182 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
183
184/** Assert that we haven't migrated CPUs when thread-context hooks are not
185 * used. */
186#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
187 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
188 ("Illegal migration! Entered on CPU %u Current %u\n", \
189 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
190
191/** Helper macro for VM-exit handlers called unexpectedly. */
192#define HMVMX_RETURN_UNEXPECTED_EXIT() \
193 do { \
194 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
195 return VERR_VMX_UNEXPECTED_EXIT; \
196 } while (0)
197
198
199/*******************************************************************************
200* Structures and Typedefs *
201*******************************************************************************/
202/**
203 * VMX transient state.
204 *
205 * A state structure for holding miscellaneous information across
206 * VMX non-root operation and restored after the transition.
207 */
208typedef struct VMXTRANSIENT
209{
210 /** The host's rflags/eflags. */
211 RTCCUINTREG uEflags;
212#if HC_ARCH_BITS == 32
213 uint32_t u32Alignment0;
214#endif
215 /** The guest's TPR value used for TPR shadowing. */
216 uint8_t u8GuestTpr;
217 /** Alignment. */
218 uint8_t abAlignment0[7];
219
220 /** The basic VM-exit reason. */
221 uint16_t uExitReason;
222 /** Alignment. */
223 uint16_t u16Alignment0;
224 /** The VM-exit interruption error code. */
225 uint32_t uExitIntErrorCode;
226 /** The VM-exit exit qualification. */
227 uint64_t uExitQualification;
228
229 /** The VM-exit interruption-information field. */
230 uint32_t uExitIntInfo;
231 /** The VM-exit instruction-length field. */
232 uint32_t cbInstr;
233 /** The VM-exit instruction-information field. */
234 union
235 {
236 /** Plain unsigned int representation. */
237 uint32_t u;
238 /** INS and OUTS information. */
239 struct
240 {
241 uint32_t u6Reserved0 : 7;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u5Reserved1 : 5;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 uint32_t uReserved2 : 14;
248 } StrIo;
249 } ExitInstrInfo;
250 /** Whether the VM-entry failed or not. */
251 bool fVMEntryFailed;
252 /** Alignment. */
253 uint8_t abAlignment1[3];
254
255 /** The VM-entry interruption-information field. */
256 uint32_t uEntryIntInfo;
257 /** The VM-entry exception error code field. */
258 uint32_t uEntryXcptErrorCode;
259 /** The VM-entry instruction length field. */
260 uint32_t cbEntryInstr;
261
262 /** IDT-vectoring information field. */
263 uint32_t uIdtVectoringInfo;
264 /** IDT-vectoring error code. */
265 uint32_t uIdtVectoringErrorCode;
266
267 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
268 uint32_t fVmcsFieldsRead;
269
270 /** Whether the guest FPU was active at the time of VM-exit. */
271 bool fWasGuestFPUStateActive;
272 /** Whether the guest debug state was active at the time of VM-exit. */
273 bool fWasGuestDebugStateActive;
274 /** Whether the hyper debug state was active at the time of VM-exit. */
275 bool fWasHyperDebugStateActive;
276 /** Whether TSC-offsetting should be setup before VM-entry. */
277 bool fUpdateTscOffsettingAndPreemptTimer;
278 /** Whether the VM-exit was caused by a page-fault during delivery of a
279 * contributory exception or a page-fault. */
280 bool fVectoringPF;
281} VMXTRANSIENT;
282AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
283AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
284AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
285AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
286AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
287/** Pointer to VMX transient state. */
288typedef VMXTRANSIENT *PVMXTRANSIENT;
289
290
291/**
292 * MSR-bitmap read permissions.
293 */
294typedef enum VMXMSREXITREAD
295{
296 /** Reading this MSR causes a VM-exit. */
297 VMXMSREXIT_INTERCEPT_READ = 0xb,
298 /** Reading this MSR does not cause a VM-exit. */
299 VMXMSREXIT_PASSTHRU_READ
300} VMXMSREXITREAD;
301/** Pointer to MSR-bitmap read permissions. */
302typedef VMXMSREXITREAD* PVMXMSREXITREAD;
303
304/**
305 * MSR-bitmap write permissions.
306 */
307typedef enum VMXMSREXITWRITE
308{
309 /** Writing to this MSR causes a VM-exit. */
310 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
311 /** Writing to this MSR does not cause a VM-exit. */
312 VMXMSREXIT_PASSTHRU_WRITE
313} VMXMSREXITWRITE;
314/** Pointer to MSR-bitmap write permissions. */
315typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
316
317
318/**
319 * VMX VM-exit handler.
320 *
321 * @returns VBox status code.
322 * @param pVCpu Pointer to the VMCPU.
323 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
324 * out-of-sync. Make sure to update the required
325 * fields before using them.
326 * @param pVmxTransient Pointer to the VMX-transient structure.
327 */
328#ifndef HMVMX_USE_FUNCTION_TABLE
329typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
330#else
331typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
332/** Pointer to VM-exit handler. */
333typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
334#endif
335
336
337/*******************************************************************************
338* Internal Functions *
339*******************************************************************************/
340static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
341static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
342static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
343 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
344#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
345static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
346#endif
347#ifndef HMVMX_USE_FUNCTION_TABLE
348DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
349# define HMVMX_EXIT_DECL static int
350#else
351# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
352#endif
353
354/** @name VM-exit handlers.
355 * @{
356 */
357static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
358static FNVMXEXITHANDLER hmR0VmxExitExtInt;
359static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
360static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
361static FNVMXEXITHANDLER hmR0VmxExitSipi;
362static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
363static FNVMXEXITHANDLER hmR0VmxExitSmi;
364static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
365static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
366static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
367static FNVMXEXITHANDLER hmR0VmxExitCpuid;
368static FNVMXEXITHANDLER hmR0VmxExitGetsec;
369static FNVMXEXITHANDLER hmR0VmxExitHlt;
370static FNVMXEXITHANDLER hmR0VmxExitInvd;
371static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
372static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
373static FNVMXEXITHANDLER hmR0VmxExitVmcall;
374static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
375static FNVMXEXITHANDLER hmR0VmxExitRsm;
376static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
377static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
378static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
379static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
380static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
381static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
382static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
383static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
384static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
385static FNVMXEXITHANDLER hmR0VmxExitMwait;
386static FNVMXEXITHANDLER hmR0VmxExitMtf;
387static FNVMXEXITHANDLER hmR0VmxExitMonitor;
388static FNVMXEXITHANDLER hmR0VmxExitPause;
389static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
390static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
391static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
392static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
393static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
394static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
395static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
396static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
397static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
398static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
399static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
400static FNVMXEXITHANDLER hmR0VmxExitRdrand;
401static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
402/** @} */
403
404static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
407static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
409static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
410#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
411static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
412#endif
413static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
414
415/*******************************************************************************
416* Global Variables *
417*******************************************************************************/
418#ifdef HMVMX_USE_FUNCTION_TABLE
419
420/**
421 * VMX_EXIT dispatch table.
422 */
423static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
424{
425 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
426 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
427 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
428 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
429 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
430 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
431 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
432 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
433 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
434 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
435 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
436 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
437 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
438 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
439 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
440 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
441 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
442 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
443 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
444 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
445 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
446 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
447 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
448 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
449 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
450 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
451 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
452 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
453 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
454 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
455 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
456 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
457 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
458 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
459 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
460 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
461 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
462 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
463 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
464 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
465 /* 40 UNDEFINED */ hmR0VmxExitPause,
466 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
467 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
468 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
469 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
470 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
471 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
472 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
473 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
474 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
475 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
476 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
477 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
478 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
479 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
480 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
481 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
482 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
483 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
484 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
485};
486#endif /* HMVMX_USE_FUNCTION_TABLE */
487
488#ifdef VBOX_STRICT
489static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
490{
491 /* 0 */ "(Not Used)",
492 /* 1 */ "VMCALL executed in VMX root operation.",
493 /* 2 */ "VMCLEAR with invalid physical address.",
494 /* 3 */ "VMCLEAR with VMXON pointer.",
495 /* 4 */ "VMLAUNCH with non-clear VMCS.",
496 /* 5 */ "VMRESUME with non-launched VMCS.",
497 /* 6 */ "VMRESUME after VMXOFF",
498 /* 7 */ "VM entry with invalid control fields.",
499 /* 8 */ "VM entry with invalid host state fields.",
500 /* 9 */ "VMPTRLD with invalid physical address.",
501 /* 10 */ "VMPTRLD with VMXON pointer.",
502 /* 11 */ "VMPTRLD with incorrect revision identifier.",
503 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
504 /* 13 */ "VMWRITE to read-only VMCS component.",
505 /* 14 */ "(Not Used)",
506 /* 15 */ "VMXON executed in VMX root operation.",
507 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
508 /* 17 */ "VM entry with non-launched executing VMCS.",
509 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
510 /* 19 */ "VMCALL with non-clear VMCS.",
511 /* 20 */ "VMCALL with invalid VM-exit control fields.",
512 /* 21 */ "(Not Used)",
513 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
514 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
515 /* 24 */ "VMCALL with invalid SMM-monitor features.",
516 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
517 /* 26 */ "VM entry with events blocked by MOV SS.",
518 /* 27 */ "(Not Used)",
519 /* 28 */ "Invalid operand to INVEPT/INVVPID."
520};
521#endif /* VBOX_STRICT */
522
523
524
525/**
526 * Updates the VM's last error record. If there was a VMX instruction error,
527 * reads the error data from the VMCS and updates VCPU's last error record as
528 * well.
529 *
530 * @param pVM Pointer to the VM.
531 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
532 * VERR_VMX_UNABLE_TO_START_VM or
533 * VERR_VMX_INVALID_VMCS_FIELD).
534 * @param rc The error code.
535 */
536static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
537{
538 AssertPtr(pVM);
539 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
540 || rc == VERR_VMX_UNABLE_TO_START_VM)
541 {
542 AssertPtrReturnVoid(pVCpu);
543 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
544 }
545 pVM->hm.s.lLastError = rc;
546}
547
548
549/**
550 * Reads the VM-entry interruption-information field from the VMCS into the VMX
551 * transient structure.
552 *
553 * @returns VBox status code.
554 * @param pVmxTransient Pointer to the VMX transient structure.
555 *
556 * @remarks No-long-jump zone!!!
557 */
558DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
559{
560 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
561 AssertRCReturn(rc, rc);
562 return VINF_SUCCESS;
563}
564
565
566/**
567 * Reads the VM-entry exception error code field from the VMCS into
568 * the VMX transient structure.
569 *
570 * @returns VBox status code.
571 * @param pVmxTransient Pointer to the VMX transient structure.
572 *
573 * @remarks No-long-jump zone!!!
574 */
575DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
576{
577 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
578 AssertRCReturn(rc, rc);
579 return VINF_SUCCESS;
580}
581
582
583/**
584 * Reads the VM-entry exception error code field from the VMCS into
585 * the VMX transient structure.
586 *
587 * @returns VBox status code.
588 * @param pVmxTransient Pointer to the VMX transient structure.
589 *
590 * @remarks No-long-jump zone!!!
591 */
592DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
593{
594 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
595 AssertRCReturn(rc, rc);
596 return VINF_SUCCESS;
597}
598
599
600/**
601 * Reads the VM-exit interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 */
607DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
608{
609 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
610 {
611 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
612 AssertRCReturn(rc, rc);
613 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
614 }
615 return VINF_SUCCESS;
616}
617
618
619/**
620 * Reads the VM-exit interruption error code from the VMCS into the VMX
621 * transient structure.
622 *
623 * @returns VBox status code.
624 * @param pVmxTransient Pointer to the VMX transient structure.
625 */
626DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
627{
628 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
629 {
630 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
631 AssertRCReturn(rc, rc);
632 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
633 }
634 return VINF_SUCCESS;
635}
636
637
638/**
639 * Reads the VM-exit instruction length field from the VMCS into the VMX
640 * transient structure.
641 *
642 * @returns VBox status code.
643 * @param pVCpu Pointer to the VMCPU.
644 * @param pVmxTransient Pointer to the VMX transient structure.
645 */
646DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
647{
648 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
649 {
650 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
651 AssertRCReturn(rc, rc);
652 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
653 }
654 return VINF_SUCCESS;
655}
656
657
658/**
659 * Reads the VM-exit instruction-information field from the VMCS into
660 * the VMX transient structure.
661 *
662 * @returns VBox status code.
663 * @param pVmxTransient Pointer to the VMX transient structure.
664 */
665DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
666{
667 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
668 {
669 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
670 AssertRCReturn(rc, rc);
671 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
672 }
673 return VINF_SUCCESS;
674}
675
676
677/**
678 * Reads the exit qualification from the VMCS into the VMX transient structure.
679 *
680 * @returns VBox status code.
681 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
682 * case).
683 * @param pVmxTransient Pointer to the VMX transient structure.
684 */
685DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
686{
687 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
688 {
689 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
690 AssertRCReturn(rc, rc);
691 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
692 }
693 return VINF_SUCCESS;
694}
695
696
697/**
698 * Reads the IDT-vectoring information field from the VMCS into the VMX
699 * transient structure.
700 *
701 * @returns VBox status code.
702 * @param pVmxTransient Pointer to the VMX transient structure.
703 *
704 * @remarks No-long-jump zone!!!
705 */
706DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
707{
708 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
709 {
710 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
711 AssertRCReturn(rc, rc);
712 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
713 }
714 return VINF_SUCCESS;
715}
716
717
718/**
719 * Reads the IDT-vectoring error code from the VMCS into the VMX
720 * transient structure.
721 *
722 * @returns VBox status code.
723 * @param pVmxTransient Pointer to the VMX transient structure.
724 */
725DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
726{
727 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
728 {
729 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
730 AssertRCReturn(rc, rc);
731 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
732 }
733 return VINF_SUCCESS;
734}
735
736
737/**
738 * Enters VMX root mode operation on the current CPU.
739 *
740 * @returns VBox status code.
741 * @param pVM Pointer to the VM (optional, can be NULL, after
742 * a resume).
743 * @param HCPhysCpuPage Physical address of the VMXON region.
744 * @param pvCpuPage Pointer to the VMXON region.
745 */
746static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
747{
748 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
749 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
750 Assert(pvCpuPage);
751 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
752
753 if (pVM)
754 {
755 /* Write the VMCS revision dword to the VMXON region. */
756 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
757 }
758
759 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
760 RTCCUINTREG uEflags = ASMIntDisableFlags();
761
762 /* Enable the VMX bit in CR4 if necessary. */
763 RTCCUINTREG uCr4 = ASMGetCR4();
764 if (!(uCr4 & X86_CR4_VMXE))
765 ASMSetCR4(uCr4 | X86_CR4_VMXE);
766
767 /* Enter VMX root mode. */
768 int rc = VMXEnable(HCPhysCpuPage);
769 if (RT_FAILURE(rc))
770 ASMSetCR4(uCr4);
771
772 /* Restore interrupts. */
773 ASMSetFlags(uEflags);
774 return rc;
775}
776
777
778/**
779 * Exits VMX root mode operation on the current CPU.
780 *
781 * @returns VBox status code.
782 */
783static int hmR0VmxLeaveRootMode(void)
784{
785 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
786
787 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
788 RTCCUINTREG uEflags = ASMIntDisableFlags();
789
790 /* If we're for some reason not in VMX root mode, then don't leave it. */
791 RTCCUINTREG uHostCR4 = ASMGetCR4();
792
793 int rc;
794 if (uHostCR4 & X86_CR4_VMXE)
795 {
796 /* Exit VMX root mode and clear the VMX bit in CR4. */
797 VMXDisable();
798 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
799 rc = VINF_SUCCESS;
800 }
801 else
802 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
803
804 /* Restore interrupts. */
805 ASMSetFlags(uEflags);
806 return rc;
807}
808
809
810/**
811 * Allocates and maps one physically contiguous page. The allocated page is
812 * zero'd out. (Used by various VT-x structures).
813 *
814 * @returns IPRT status code.
815 * @param pMemObj Pointer to the ring-0 memory object.
816 * @param ppVirt Where to store the virtual address of the
817 * allocation.
818 * @param pPhys Where to store the physical address of the
819 * allocation.
820 */
821DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
822{
823 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
824 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
825 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
826
827 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
828 if (RT_FAILURE(rc))
829 return rc;
830 *ppVirt = RTR0MemObjAddress(*pMemObj);
831 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
832 ASMMemZero32(*ppVirt, PAGE_SIZE);
833 return VINF_SUCCESS;
834}
835
836
837/**
838 * Frees and unmaps an allocated physical page.
839 *
840 * @param pMemObj Pointer to the ring-0 memory object.
841 * @param ppVirt Where to re-initialize the virtual address of
842 * allocation as 0.
843 * @param pHCPhys Where to re-initialize the physical address of the
844 * allocation as 0.
845 */
846DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
847{
848 AssertPtr(pMemObj);
849 AssertPtr(ppVirt);
850 AssertPtr(pHCPhys);
851 if (*pMemObj != NIL_RTR0MEMOBJ)
852 {
853 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
854 AssertRC(rc);
855 *pMemObj = NIL_RTR0MEMOBJ;
856 *ppVirt = 0;
857 *pHCPhys = 0;
858 }
859}
860
861
862/**
863 * Worker function to free VT-x related structures.
864 *
865 * @returns IPRT status code.
866 * @param pVM Pointer to the VM.
867 */
868static void hmR0VmxStructsFree(PVM pVM)
869{
870 for (VMCPUID i = 0; i < pVM->cCpus; i++)
871 {
872 PVMCPU pVCpu = &pVM->aCpus[i];
873 AssertPtr(pVCpu);
874
875 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
876 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
877
878 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
879 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
880
881 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
882 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
883 }
884
885 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
886#ifdef VBOX_WITH_CRASHDUMP_MAGIC
887 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
888#endif
889}
890
891
892/**
893 * Worker function to allocate VT-x related VM structures.
894 *
895 * @returns IPRT status code.
896 * @param pVM Pointer to the VM.
897 */
898static int hmR0VmxStructsAlloc(PVM pVM)
899{
900 /*
901 * Initialize members up-front so we can cleanup properly on allocation failure.
902 */
903#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
904 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
905 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
906 pVM->hm.s.vmx.HCPhys##a_Name = 0;
907
908#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
909 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
910 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
911 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
912
913#ifdef VBOX_WITH_CRASHDUMP_MAGIC
914 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
915#endif
916 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
917
918 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
919 for (VMCPUID i = 0; i < pVM->cCpus; i++)
920 {
921 PVMCPU pVCpu = &pVM->aCpus[i];
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
923 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
924 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
925 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
926 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
927 }
928#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
929#undef VMXLOCAL_INIT_VM_MEMOBJ
930
931 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
932 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
933 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
934 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
935
936 /*
937 * Allocate all the VT-x structures.
938 */
939 int rc = VINF_SUCCESS;
940#ifdef VBOX_WITH_CRASHDUMP_MAGIC
941 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
942 if (RT_FAILURE(rc))
943 goto cleanup;
944 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
945 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
946#endif
947
948 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
949 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
950 {
951 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
952 &pVM->hm.s.vmx.HCPhysApicAccess);
953 if (RT_FAILURE(rc))
954 goto cleanup;
955 }
956
957 /*
958 * Initialize per-VCPU VT-x structures.
959 */
960 for (VMCPUID i = 0; i < pVM->cCpus; i++)
961 {
962 PVMCPU pVCpu = &pVM->aCpus[i];
963 AssertPtr(pVCpu);
964
965 /* Allocate the VM control structure (VMCS). */
966 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
967 if (RT_FAILURE(rc))
968 goto cleanup;
969
970 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
971 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
972 {
973 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
974 &pVCpu->hm.s.vmx.HCPhysVirtApic);
975 if (RT_FAILURE(rc))
976 goto cleanup;
977 }
978
979 /*
980 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
981 * transparent accesses of specific MSRs.
982 *
983 * If the condition for enabling MSR bitmaps changes here, don't forget to
984 * update HMIsMsrBitmapsAvailable().
985 */
986 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
987 {
988 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
989 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
990 if (RT_FAILURE(rc))
991 goto cleanup;
992 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
993 }
994
995 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
996 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
997 if (RT_FAILURE(rc))
998 goto cleanup;
999
1000 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1001 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1002 if (RT_FAILURE(rc))
1003 goto cleanup;
1004 }
1005
1006 return VINF_SUCCESS;
1007
1008cleanup:
1009 hmR0VmxStructsFree(pVM);
1010 return rc;
1011}
1012
1013
1014/**
1015 * Does global VT-x initialization (called during module initialization).
1016 *
1017 * @returns VBox status code.
1018 */
1019VMMR0DECL(int) VMXR0GlobalInit(void)
1020{
1021#ifdef HMVMX_USE_FUNCTION_TABLE
1022 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1023# ifdef VBOX_STRICT
1024 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1025 Assert(g_apfnVMExitHandlers[i]);
1026# endif
1027#endif
1028 return VINF_SUCCESS;
1029}
1030
1031
1032/**
1033 * Does global VT-x termination (called during module termination).
1034 */
1035VMMR0DECL(void) VMXR0GlobalTerm()
1036{
1037 /* Nothing to do currently. */
1038}
1039
1040
1041/**
1042 * Sets up and activates VT-x on the current CPU.
1043 *
1044 * @returns VBox status code.
1045 * @param pCpu Pointer to the global CPU info struct.
1046 * @param pVM Pointer to the VM (can be NULL after a host resume
1047 * operation).
1048 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1049 * fEnabledByHost is true).
1050 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1051 * @a fEnabledByHost is true).
1052 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1053 * enable VT-x on the host.
1054 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1055 */
1056VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1057 void *pvMsrs)
1058{
1059 Assert(pCpu);
1060 Assert(pvMsrs);
1061 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1062
1063 /* Enable VT-x if it's not already enabled by the host. */
1064 if (!fEnabledByHost)
1065 {
1066 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1067 if (RT_FAILURE(rc))
1068 return rc;
1069 }
1070
1071 /*
1072 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1073 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1074 */
1075 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1076 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1077 {
1078 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1079 pCpu->fFlushAsidBeforeUse = false;
1080 }
1081 else
1082 pCpu->fFlushAsidBeforeUse = true;
1083
1084 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1085 ++pCpu->cTlbFlushes;
1086
1087 return VINF_SUCCESS;
1088}
1089
1090
1091/**
1092 * Deactivates VT-x on the current CPU.
1093 *
1094 * @returns VBox status code.
1095 * @param pCpu Pointer to the global CPU info struct.
1096 * @param pvCpuPage Pointer to the VMXON region.
1097 * @param HCPhysCpuPage Physical address of the VMXON region.
1098 *
1099 * @remarks This function should never be called when SUPR0EnableVTx() or
1100 * similar was used to enable VT-x on the host.
1101 */
1102VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1103{
1104 NOREF(pCpu);
1105 NOREF(pvCpuPage);
1106 NOREF(HCPhysCpuPage);
1107
1108 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1109 return hmR0VmxLeaveRootMode();
1110}
1111
1112
1113/**
1114 * Sets the permission bits for the specified MSR in the MSR bitmap.
1115 *
1116 * @param pVCpu Pointer to the VMCPU.
1117 * @param uMSR The MSR value.
1118 * @param enmRead Whether reading this MSR causes a VM-exit.
1119 * @param enmWrite Whether writing this MSR causes a VM-exit.
1120 */
1121static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1122{
1123 int32_t iBit;
1124 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1125
1126 /*
1127 * Layout:
1128 * 0x000 - 0x3ff - Low MSR read bits
1129 * 0x400 - 0x7ff - High MSR read bits
1130 * 0x800 - 0xbff - Low MSR write bits
1131 * 0xc00 - 0xfff - High MSR write bits
1132 */
1133 if (uMsr <= 0x00001FFF)
1134 iBit = uMsr;
1135 else if ( uMsr >= 0xC0000000
1136 && uMsr <= 0xC0001FFF)
1137 {
1138 iBit = (uMsr - 0xC0000000);
1139 pbMsrBitmap += 0x400;
1140 }
1141 else
1142 {
1143 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1144 return;
1145 }
1146
1147 Assert(iBit <= 0x1fff);
1148 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1149 ASMBitSet(pbMsrBitmap, iBit);
1150 else
1151 ASMBitClear(pbMsrBitmap, iBit);
1152
1153 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1154 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1155 else
1156 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1157}
1158
1159
1160#ifdef VBOX_STRICT
1161/**
1162 * Gets the permission bits for the specified MSR in the MSR bitmap.
1163 *
1164 * @returns VBox status code.
1165 * @retval VINF_SUCCESS if the specified MSR is found.
1166 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1167 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1168 *
1169 * @param pVCpu Pointer to the VMCPU.
1170 * @param uMsr The MSR.
1171 * @param penmRead Where to store the read permissions.
1172 * @param penmWrite Where to store the write permissions.
1173 */
1174static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1175{
1176 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1177 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1178 int32_t iBit;
1179 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1180
1181 /* See hmR0VmxSetMsrPermission() for the layout. */
1182 if (uMsr <= 0x00001FFF)
1183 iBit = uMsr;
1184 else if ( uMsr >= 0xC0000000
1185 && uMsr <= 0xC0001FFF)
1186 {
1187 iBit = (uMsr - 0xC0000000);
1188 pbMsrBitmap += 0x400;
1189 }
1190 else
1191 {
1192 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1193 return VERR_NOT_SUPPORTED;
1194 }
1195
1196 Assert(iBit <= 0x1fff);
1197 if (ASMBitTest(pbMsrBitmap, iBit))
1198 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1199 else
1200 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1201
1202 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1203 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1204 else
1205 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1206 return VINF_SUCCESS;
1207}
1208#endif /* VBOX_STRICT */
1209
1210
1211/**
1212 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1213 * area.
1214 *
1215 * @returns VBox status code.
1216 * @param pVCpu Pointer to the VMCPU.
1217 * @param cMsrs The number of MSRs.
1218 */
1219DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1220{
1221 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1222 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1223 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1224 {
1225 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1226 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1227 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1228 }
1229
1230 /* Update number of guest MSRs to load/store across the world-switch. */
1231 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1232 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1233
1234 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1235 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1236
1237 /* Update the VCPU's copy of the MSR count. */
1238 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1239
1240 return VINF_SUCCESS;
1241}
1242
1243
1244/**
1245 * Adds a new (or updates the value of an existing) guest/host MSR
1246 * pair to be swapped during the world-switch as part of the
1247 * auto-load/store MSR area in the VMCS.
1248 *
1249 * @returns true if the MSR was added -and- its value was updated, false
1250 * otherwise.
1251 * @param pVCpu Pointer to the VMCPU.
1252 * @param uMsr The MSR.
1253 * @param uGuestMsr Value of the guest MSR.
1254 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1255 * necessary.
1256 */
1257static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1258{
1259 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1260 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1261 uint32_t i;
1262 for (i = 0; i < cMsrs; i++)
1263 {
1264 if (pGuestMsr->u32Msr == uMsr)
1265 break;
1266 pGuestMsr++;
1267 }
1268
1269 bool fAdded = false;
1270 if (i == cMsrs)
1271 {
1272 ++cMsrs;
1273 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1274 AssertRC(rc);
1275
1276 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1277 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1278 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1279
1280 fAdded = true;
1281 }
1282
1283 /* Update the MSR values in the auto-load/store MSR area. */
1284 pGuestMsr->u32Msr = uMsr;
1285 pGuestMsr->u64Value = uGuestMsrValue;
1286
1287 /* Create/update the MSR slot in the host MSR area. */
1288 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1289 pHostMsr += i;
1290 pHostMsr->u32Msr = uMsr;
1291
1292 /*
1293 * Update the host MSR only when requested by the caller AND when we're
1294 * adding it to the auto-load/store area. Otherwise, it would have been
1295 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1296 */
1297 bool fUpdatedMsrValue = false;
1298 if ( fAdded
1299 && fUpdateHostMsr)
1300 {
1301 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1302 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1303 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1304 fUpdatedMsrValue = true;
1305 }
1306
1307 return fUpdatedMsrValue;
1308}
1309
1310
1311/**
1312 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1313 * auto-load/store MSR area in the VMCS.
1314 *
1315 * @returns VBox status code.
1316 * @param pVCpu Pointer to the VMCPU.
1317 * @param uMsr The MSR.
1318 */
1319static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1320{
1321 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1322 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1323 for (uint32_t i = 0; i < cMsrs; i++)
1324 {
1325 /* Find the MSR. */
1326 if (pGuestMsr->u32Msr == uMsr)
1327 {
1328 /* If it's the last MSR, simply reduce the count. */
1329 if (i == cMsrs - 1)
1330 {
1331 --cMsrs;
1332 break;
1333 }
1334
1335 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1336 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1337 pLastGuestMsr += cMsrs - 1;
1338 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1339 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1340
1341 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1342 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 pLastHostMsr += cMsrs - 1;
1344 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1345 pHostMsr->u64Value = pLastHostMsr->u64Value;
1346 --cMsrs;
1347 break;
1348 }
1349 pGuestMsr++;
1350 }
1351
1352 /* Update the VMCS if the count changed (meaning the MSR was found). */
1353 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1354 {
1355 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1356 AssertRCReturn(rc, rc);
1357
1358 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1359 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1360 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1361
1362 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1363 return VINF_SUCCESS;
1364 }
1365
1366 return VERR_NOT_FOUND;
1367}
1368
1369
1370/**
1371 * Checks if the specified guest MSR is part of the auto-load/store area in
1372 * the VMCS.
1373 *
1374 * @returns true if found, false otherwise.
1375 * @param pVCpu Pointer to the VMCPU.
1376 * @param uMsr The MSR to find.
1377 */
1378static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1379{
1380 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1381 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1382
1383 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1384 {
1385 if (pGuestMsr->u32Msr == uMsr)
1386 return true;
1387 }
1388 return false;
1389}
1390
1391
1392/**
1393 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1394 *
1395 * @param pVCpu Pointer to the VMCPU.
1396 *
1397 * @remarks No-long-jump zone!!!
1398 */
1399static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1400{
1401 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1402 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1403 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1404 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1405
1406 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1407 {
1408 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1409
1410 /*
1411 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1412 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1413 */
1414 if (pHostMsr->u32Msr == MSR_K6_EFER)
1415 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1416 else
1417 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1418 }
1419
1420 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1421}
1422
1423
1424#if HC_ARCH_BITS == 64
1425/**
1426 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1427 * perform lazy restoration of the host MSRs while leaving VT-x.
1428 *
1429 * @param pVCpu Pointer to the VMCPU.
1430 *
1431 * @remarks No-long-jump zone!!!
1432 */
1433static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1434{
1435 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1436
1437 /*
1438 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1439 */
1440 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1441 {
1442 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1443 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1444 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1445 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1446 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1447 }
1448}
1449
1450
1451/**
1452 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1453 * lazily while leaving VT-x.
1454 *
1455 * @returns true if it does, false otherwise.
1456 * @param pVCpu Pointer to the VMCPU.
1457 * @param uMsr The MSR to check.
1458 */
1459static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1460{
1461 NOREF(pVCpu);
1462 switch (uMsr)
1463 {
1464 case MSR_K8_LSTAR:
1465 case MSR_K6_STAR:
1466 case MSR_K8_SF_MASK:
1467 case MSR_K8_KERNEL_GS_BASE:
1468 return true;
1469 }
1470 return false;
1471}
1472
1473
1474/**
1475 * Saves a set of guests MSRs back into the guest-CPU context.
1476 *
1477 * @param pVCpu Pointer to the VMCPU.
1478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1479 * out-of-sync. Make sure to update the required fields
1480 * before using them.
1481 *
1482 * @remarks No-long-jump zone!!!
1483 */
1484static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1485{
1486 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1487 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1488
1489 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1490 {
1491 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1492 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1493 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1494 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1495 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1496 }
1497}
1498
1499
1500/**
1501 * Loads a set of guests MSRs to allow read/passthru to the guest.
1502 *
1503 * The name of this function is slightly confusing. This function does NOT
1504 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1505 * common prefix for functions dealing with "lazy restoration" of the shared
1506 * MSRs.
1507 *
1508 * @param pVCpu Pointer to the VMCPU.
1509 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1510 * out-of-sync. Make sure to update the required fields
1511 * before using them.
1512 *
1513 * @remarks No-long-jump zone!!!
1514 */
1515static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1516{
1517 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1518 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1519
1520 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1521 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1522 {
1523#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1524 do { \
1525 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1526 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1527 else \
1528 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1529 } while (0)
1530
1531 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1532 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1533 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1534 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1535#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1536 }
1537 else
1538 {
1539 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1540 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1541 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1542 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1543 }
1544 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1545}
1546
1547
1548/**
1549 * Performs lazy restoration of the set of host MSRs if they were previously
1550 * loaded with guest MSR values.
1551 *
1552 * @param pVCpu Pointer to the VMCPU.
1553 *
1554 * @remarks No-long-jump zone!!!
1555 * @remarks The guest MSRs should have been saved back into the guest-CPU
1556 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1557 */
1558static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1559{
1560 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1561 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1562
1563 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1564 {
1565 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1566 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1567 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1568 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1570 }
1571 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1572}
1573#endif /* HC_ARCH_BITS == 64 */
1574
1575
1576/**
1577 * Verifies that our cached values of the VMCS controls are all
1578 * consistent with what's actually present in the VMCS.
1579 *
1580 * @returns VBox status code.
1581 * @param pVCpu Pointer to the VMCPU.
1582 */
1583static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1584{
1585 uint32_t u32Val;
1586 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1587 AssertRCReturn(rc, rc);
1588 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1589 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1590
1591 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1592 AssertRCReturn(rc, rc);
1593 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1594 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1595
1596 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1597 AssertRCReturn(rc, rc);
1598 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1599 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1600
1601 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1602 AssertRCReturn(rc, rc);
1603 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1604 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1605
1606 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1607 AssertRCReturn(rc, rc);
1608 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1609 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1610
1611 return VINF_SUCCESS;
1612}
1613
1614
1615#ifdef VBOX_STRICT
1616/**
1617 * Verifies that our cached host EFER value has not changed
1618 * since we cached it.
1619 *
1620 * @param pVCpu Pointer to the VMCPU.
1621 */
1622static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1623{
1624 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1625
1626 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1627 {
1628 uint64_t u64Val;
1629 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1630 AssertRC(rc);
1631
1632 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1633 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1634 }
1635}
1636
1637
1638/**
1639 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1640 * VMCS are correct.
1641 *
1642 * @param pVCpu Pointer to the VMCPU.
1643 */
1644static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1645{
1646 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1647
1648 /* Verify MSR counts in the VMCS are what we think it should be. */
1649 uint32_t cMsrs;
1650 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1651 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1652
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1654 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1655
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1657 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1658
1659 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1660 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1661 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1662 {
1663 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1664 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1665 pGuestMsr->u32Msr, cMsrs));
1666
1667 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1668 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1669 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1670
1671 /* Verify that the permissions are as expected in the MSR bitmap. */
1672 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1673 {
1674 VMXMSREXITREAD enmRead;
1675 VMXMSREXITWRITE enmWrite;
1676 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1677 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1678 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1679 {
1680 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1681 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1682 }
1683 else
1684 {
1685 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1686 pGuestMsr->u32Msr, cMsrs));
1687 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1688 pGuestMsr->u32Msr, cMsrs));
1689 }
1690 }
1691 }
1692}
1693#endif /* VBOX_STRICT */
1694
1695
1696/**
1697 * Flushes the TLB using EPT.
1698 *
1699 * @returns VBox status code.
1700 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1701 * enmFlush).
1702 * @param enmFlush Type of flush.
1703 *
1704 * @remarks Caller is responsible for making sure this function is called only
1705 * when NestedPaging is supported and providing @a enmFlush that is
1706 * supported by the CPU.
1707 * @remarks Can be called with interrupts disabled.
1708 */
1709static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1710{
1711 uint64_t au64Descriptor[2];
1712 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1713 au64Descriptor[0] = 0;
1714 else
1715 {
1716 Assert(pVCpu);
1717 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1718 }
1719 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1720
1721 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1722 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1723 rc));
1724 if ( RT_SUCCESS(rc)
1725 && pVCpu)
1726 {
1727 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1728 }
1729}
1730
1731
1732/**
1733 * Flushes the TLB using VPID.
1734 *
1735 * @returns VBox status code.
1736 * @param pVM Pointer to the VM.
1737 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1738 * enmFlush).
1739 * @param enmFlush Type of flush.
1740 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1741 * on @a enmFlush).
1742 *
1743 * @remarks Can be called with interrupts disabled.
1744 */
1745static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1746{
1747 NOREF(pVM);
1748 AssertPtr(pVM);
1749 Assert(pVM->hm.s.vmx.fVpid);
1750
1751 uint64_t au64Descriptor[2];
1752 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1753 {
1754 au64Descriptor[0] = 0;
1755 au64Descriptor[1] = 0;
1756 }
1757 else
1758 {
1759 AssertPtr(pVCpu);
1760 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1761 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1762 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1763 au64Descriptor[1] = GCPtr;
1764 }
1765
1766 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1767 AssertMsg(rc == VINF_SUCCESS,
1768 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1769 if ( RT_SUCCESS(rc)
1770 && pVCpu)
1771 {
1772 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1773 }
1774}
1775
1776
1777/**
1778 * Invalidates a guest page by guest virtual address. Only relevant for
1779 * EPT/VPID, otherwise there is nothing really to invalidate.
1780 *
1781 * @returns VBox status code.
1782 * @param pVM Pointer to the VM.
1783 * @param pVCpu Pointer to the VMCPU.
1784 * @param GCVirt Guest virtual address of the page to invalidate.
1785 */
1786VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1787{
1788 AssertPtr(pVM);
1789 AssertPtr(pVCpu);
1790 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1791
1792 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1793 if (!fFlushPending)
1794 {
1795 /*
1796 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1797 * See @bugref{6043} and @bugref{6177}.
1798 *
1799 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1800 * function maybe called in a loop with individual addresses.
1801 */
1802 if (pVM->hm.s.vmx.fVpid)
1803 {
1804 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1805 {
1806 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1807 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1808 }
1809 else
1810 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1811 }
1812 else if (pVM->hm.s.fNestedPaging)
1813 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1814 }
1815
1816 return VINF_SUCCESS;
1817}
1818
1819
1820/**
1821 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1822 * otherwise there is nothing really to invalidate.
1823 *
1824 * @returns VBox status code.
1825 * @param pVM Pointer to the VM.
1826 * @param pVCpu Pointer to the VMCPU.
1827 * @param GCPhys Guest physical address of the page to invalidate.
1828 */
1829VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1830{
1831 NOREF(pVM); NOREF(GCPhys);
1832 LogFlowFunc(("%RGp\n", GCPhys));
1833
1834 /*
1835 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1836 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1837 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1838 */
1839 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1840 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1841 return VINF_SUCCESS;
1842}
1843
1844
1845/**
1846 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1847 * case where neither EPT nor VPID is supported by the CPU.
1848 *
1849 * @param pVM Pointer to the VM.
1850 * @param pVCpu Pointer to the VMCPU.
1851 * @param pCpu Pointer to the global HM struct.
1852 *
1853 * @remarks Called with interrupts disabled.
1854 */
1855static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1856{
1857 AssertPtr(pVCpu);
1858 AssertPtr(pCpu);
1859 NOREF(pVM);
1860
1861 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1862
1863 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1864#if 0
1865 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1866 pVCpu->hm.s.TlbShootdown.cPages = 0;
1867#endif
1868
1869 Assert(pCpu->idCpu != NIL_RTCPUID);
1870 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1871 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1872 pVCpu->hm.s.fForceTLBFlush = false;
1873 return;
1874}
1875
1876
1877/**
1878 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1879 *
1880 * @param pVM Pointer to the VM.
1881 * @param pVCpu Pointer to the VMCPU.
1882 * @param pCpu Pointer to the global HM CPU struct.
1883 * @remarks All references to "ASID" in this function pertains to "VPID" in
1884 * Intel's nomenclature. The reason is, to avoid confusion in compare
1885 * statements since the host-CPU copies are named "ASID".
1886 *
1887 * @remarks Called with interrupts disabled.
1888 */
1889static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1890{
1891#ifdef VBOX_WITH_STATISTICS
1892 bool fTlbFlushed = false;
1893# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1894# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1895 if (!fTlbFlushed) \
1896 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1897 } while (0)
1898#else
1899# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1900# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1901#endif
1902
1903 AssertPtr(pVM);
1904 AssertPtr(pCpu);
1905 AssertPtr(pVCpu);
1906 Assert(pCpu->idCpu != NIL_RTCPUID);
1907
1908 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1909 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1910 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1911
1912 /*
1913 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1914 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1915 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1916 */
1917 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1918 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1919 {
1920 ++pCpu->uCurrentAsid;
1921 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1922 {
1923 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1924 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1925 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1926 }
1927
1928 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1929 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1930 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1931
1932 /*
1933 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1934 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1935 */
1936 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1937 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1938 HMVMX_SET_TAGGED_TLB_FLUSHED();
1939 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1940 }
1941
1942 /* Check for explicit TLB shootdowns. */
1943 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1944 {
1945 /*
1946 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1947 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1948 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1949 * but not guest-physical mappings.
1950 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1951 */
1952 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1953 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1954 HMVMX_SET_TAGGED_TLB_FLUSHED();
1955 }
1956
1957 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1958 * where it is commented out. Support individual entry flushing
1959 * someday. */
1960#if 0
1961 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1962 {
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1964
1965 /*
1966 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1967 * as supported by the CPU.
1968 */
1969 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1970 {
1971 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1972 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1973 }
1974 else
1975 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1976
1977 HMVMX_SET_TAGGED_TLB_FLUSHED();
1978 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1979 pVCpu->hm.s.TlbShootdown.cPages = 0;
1980 }
1981#endif
1982
1983 pVCpu->hm.s.fForceTLBFlush = false;
1984
1985 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1986
1987 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1988 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1989 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1990 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1991 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1992 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1993 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1994 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1995 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1996
1997 /* Update VMCS with the VPID. */
1998 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1999 AssertRC(rc);
2000
2001#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2002}
2003
2004
2005/**
2006 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2007 *
2008 * @returns VBox status code.
2009 * @param pVM Pointer to the VM.
2010 * @param pVCpu Pointer to the VMCPU.
2011 * @param pCpu Pointer to the global HM CPU struct.
2012 *
2013 * @remarks Called with interrupts disabled.
2014 */
2015static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2016{
2017 AssertPtr(pVM);
2018 AssertPtr(pVCpu);
2019 AssertPtr(pCpu);
2020 Assert(pCpu->idCpu != NIL_RTCPUID);
2021 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2022 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2023
2024 /*
2025 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2026 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2027 */
2028 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2029 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2030 {
2031 pVCpu->hm.s.fForceTLBFlush = true;
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2033 }
2034
2035 /* Check for explicit TLB shootdown flushes. */
2036 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2037 {
2038 pVCpu->hm.s.fForceTLBFlush = true;
2039 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2040 }
2041
2042 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2043 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2044
2045 if (pVCpu->hm.s.fForceTLBFlush)
2046 {
2047 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2048 pVCpu->hm.s.fForceTLBFlush = false;
2049 }
2050 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2051 * where it is commented out. Support individual entry flushing
2052 * someday. */
2053#if 0
2054 else
2055 {
2056 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2057 {
2058 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2059 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2060 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2061 }
2062 else
2063 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2064
2065 pVCpu->hm.s.TlbShootdown.cPages = 0;
2066 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2067 }
2068#endif
2069}
2070
2071
2072/**
2073 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2074 *
2075 * @returns VBox status code.
2076 * @param pVM Pointer to the VM.
2077 * @param pVCpu Pointer to the VMCPU.
2078 * @param pCpu Pointer to the global HM CPU struct.
2079 *
2080 * @remarks Called with interrupts disabled.
2081 */
2082static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2083{
2084 AssertPtr(pVM);
2085 AssertPtr(pVCpu);
2086 AssertPtr(pCpu);
2087 Assert(pCpu->idCpu != NIL_RTCPUID);
2088 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2089 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2090
2091 /*
2092 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2093 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2094 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2095 */
2096 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2097 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2098 {
2099 pVCpu->hm.s.fForceTLBFlush = true;
2100 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2101 }
2102
2103 /* Check for explicit TLB shootdown flushes. */
2104 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2105 {
2106 /*
2107 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2108 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2109 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2110 */
2111 pVCpu->hm.s.fForceTLBFlush = true;
2112 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2113 }
2114
2115 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2116 if (pVCpu->hm.s.fForceTLBFlush)
2117 {
2118 ++pCpu->uCurrentAsid;
2119 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2120 {
2121 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2122 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2123 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2124 }
2125
2126 pVCpu->hm.s.fForceTLBFlush = false;
2127 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2128 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2129 if (pCpu->fFlushAsidBeforeUse)
2130 {
2131 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2132 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2133 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2134 {
2135 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2136 pCpu->fFlushAsidBeforeUse = false;
2137 }
2138 else
2139 {
2140 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2141 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2142 }
2143 }
2144 }
2145 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2146 * where it is commented out. Support individual entry flushing
2147 * someday. */
2148#if 0
2149 else
2150 {
2151 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2152 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2153 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2154 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2155
2156 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2157 {
2158 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2159 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2160 {
2161 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2162 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2163 }
2164 else
2165 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2166
2167 pVCpu->hm.s.TlbShootdown.cPages = 0;
2168 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2169 }
2170 else
2171 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2172 }
2173#endif
2174
2175 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2176 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2177 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2178 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2179 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2180 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2181 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2182
2183 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2184 AssertRC(rc);
2185}
2186
2187
2188/**
2189 * Flushes the guest TLB entry based on CPU capabilities.
2190 *
2191 * @param pVCpu Pointer to the VMCPU.
2192 * @param pCpu Pointer to the global HM CPU struct.
2193 */
2194DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2195{
2196#ifdef HMVMX_ALWAYS_FLUSH_TLB
2197 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2198#endif
2199 PVM pVM = pVCpu->CTX_SUFF(pVM);
2200 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2201 {
2202 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2203 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2204 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2205 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2206 default:
2207 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2208 break;
2209 }
2210
2211 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2212 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2213
2214 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2215}
2216
2217
2218/**
2219 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2220 * TLB entries from the host TLB before VM-entry.
2221 *
2222 * @returns VBox status code.
2223 * @param pVM Pointer to the VM.
2224 */
2225static int hmR0VmxSetupTaggedTlb(PVM pVM)
2226{
2227 /*
2228 * Determine optimal flush type for Nested Paging.
2229 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2230 * guest execution (see hmR3InitFinalizeR0()).
2231 */
2232 if (pVM->hm.s.fNestedPaging)
2233 {
2234 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2235 {
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2237 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2238 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2239 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2240 else
2241 {
2242 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2243 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245 }
2246
2247 /* Make sure the write-back cacheable memory type for EPT is supported. */
2248 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2249 {
2250 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2251 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2252 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2253 }
2254 }
2255 else
2256 {
2257 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2258 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2260 }
2261 }
2262
2263 /*
2264 * Determine optimal flush type for VPID.
2265 */
2266 if (pVM->hm.s.vmx.fVpid)
2267 {
2268 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2269 {
2270 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2271 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2272 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2273 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2274 else
2275 {
2276 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2277 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2278 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2279 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2280 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2281 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2282 pVM->hm.s.vmx.fVpid = false;
2283 }
2284 }
2285 else
2286 {
2287 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2288 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2289 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2290 pVM->hm.s.vmx.fVpid = false;
2291 }
2292 }
2293
2294 /*
2295 * Setup the handler for flushing tagged-TLBs.
2296 */
2297 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2298 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2299 else if (pVM->hm.s.fNestedPaging)
2300 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2301 else if (pVM->hm.s.vmx.fVpid)
2302 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2303 else
2304 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2305 return VINF_SUCCESS;
2306}
2307
2308
2309/**
2310 * Sets up pin-based VM-execution controls in the VMCS.
2311 *
2312 * @returns VBox status code.
2313 * @param pVM Pointer to the VM.
2314 * @param pVCpu Pointer to the VMCPU.
2315 */
2316static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2317{
2318 AssertPtr(pVM);
2319 AssertPtr(pVCpu);
2320
2321 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2322 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2323
2324 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2325 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2326
2327 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2328 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2329
2330 /* Enable the VMX preemption timer. */
2331 if (pVM->hm.s.vmx.fUsePreemptTimer)
2332 {
2333 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2334 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2335 }
2336
2337 if ((val & zap) != val)
2338 {
2339 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2340 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2341 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2342 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2343 }
2344
2345 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2346 AssertRCReturn(rc, rc);
2347
2348 /* Update VCPU with the currently set pin-based VM-execution controls. */
2349 pVCpu->hm.s.vmx.u32PinCtls = val;
2350 return rc;
2351}
2352
2353
2354/**
2355 * Sets up processor-based VM-execution controls in the VMCS.
2356 *
2357 * @returns VBox status code.
2358 * @param pVM Pointer to the VM.
2359 * @param pVMCPU Pointer to the VMCPU.
2360 */
2361static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2362{
2363 AssertPtr(pVM);
2364 AssertPtr(pVCpu);
2365
2366 int rc = VERR_INTERNAL_ERROR_5;
2367 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2368 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2369
2370 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2371 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2372 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2373 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2374 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2375 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2376 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2377
2378 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2379 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2380 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2381 {
2382 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2383 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2384 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2385 }
2386
2387 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2388 if (!pVM->hm.s.fNestedPaging)
2389 {
2390 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2391 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2392 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2394 }
2395
2396 /* Use TPR shadowing if supported by the CPU. */
2397 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2398 {
2399 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2400 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2401 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2402 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2403 AssertRCReturn(rc, rc);
2404
2405 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2406 /* CR8 writes cause a VM-exit based on TPR threshold. */
2407 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2408 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2409 }
2410 else
2411 {
2412 /*
2413 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2414 * Set this control only for 64-bit guests.
2415 */
2416 if (pVM->hm.s.fAllow64BitGuests)
2417 {
2418 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2419 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2420 }
2421 }
2422
2423 /* Use MSR-bitmaps if supported by the CPU. */
2424 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2425 {
2426 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2427
2428 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2429 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2430 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2431 AssertRCReturn(rc, rc);
2432
2433 /*
2434 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2435 * automatically using dedicated fields in the VMCS.
2436 */
2437 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2438 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2439 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2440 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2441 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2442
2443#if HC_ARCH_BITS == 64
2444 /*
2445 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2446 */
2447 if (pVM->hm.s.fAllow64BitGuests)
2448 {
2449 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2450 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2451 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2452 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2453 }
2454#endif
2455 }
2456
2457 /* If we're using virtual NMIs, we need the NMI-window exiting feature. */
2458 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2459 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
2460 {
2461 val |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
2462 }
2463
2464 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2465 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2466 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2467
2468 if ((val & zap) != val)
2469 {
2470 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2471 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2472 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2473 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2474 }
2475
2476 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2477 AssertRCReturn(rc, rc);
2478
2479 /* Update VCPU with the currently set processor-based VM-execution controls. */
2480 pVCpu->hm.s.vmx.u32ProcCtls = val;
2481
2482 /*
2483 * Secondary processor-based VM-execution controls.
2484 */
2485 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2486 {
2487 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2488 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2489
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2491 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2492
2493 if (pVM->hm.s.fNestedPaging)
2494 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2495 else
2496 {
2497 /*
2498 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2499 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2500 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2501 */
2502 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2503 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2504 }
2505
2506 if (pVM->hm.s.vmx.fVpid)
2507 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2508
2509 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2510 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2511
2512 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2513 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2514 * done dynamically. */
2515 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2516 {
2517 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2518 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2519 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2520 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2521 AssertRCReturn(rc, rc);
2522 }
2523
2524 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2525 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2526
2527 if ((val & zap) != val)
2528 {
2529 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2530 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2531 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2532 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2533 }
2534
2535 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2536 AssertRCReturn(rc, rc);
2537
2538 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2539 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2540 }
2541 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2542 {
2543 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2544 "available\n"));
2545 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2546 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2547 }
2548
2549 return VINF_SUCCESS;
2550}
2551
2552
2553/**
2554 * Sets up miscellaneous (everything other than Pin & Processor-based
2555 * VM-execution) control fields in the VMCS.
2556 *
2557 * @returns VBox status code.
2558 * @param pVM Pointer to the VM.
2559 * @param pVCpu Pointer to the VMCPU.
2560 */
2561static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2562{
2563 NOREF(pVM);
2564 AssertPtr(pVM);
2565 AssertPtr(pVCpu);
2566
2567 int rc = VERR_GENERAL_FAILURE;
2568
2569 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2570#if 0
2571 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2572 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2573 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2574
2575 /*
2576 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2577 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2578 * We thus use the exception bitmap to control it rather than use both.
2579 */
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2582
2583 /** @todo Explore possibility of using IO-bitmaps. */
2584 /* All IO & IOIO instructions cause VM-exits. */
2585 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2586 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2587
2588 /* Initialize the MSR-bitmap area. */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2590 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2592#endif
2593
2594 /* Setup MSR auto-load/store area. */
2595 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2596 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2597 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2598 AssertRCReturn(rc, rc);
2599 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2600 AssertRCReturn(rc, rc);
2601
2602 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2605 AssertRCReturn(rc, rc);
2606
2607 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2609 AssertRCReturn(rc, rc);
2610
2611 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2612#if 0
2613 /* Setup debug controls */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2615 AssertRCReturn(rc, rc);
2616 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2617 AssertRCReturn(rc, rc);
2618#endif
2619
2620 return rc;
2621}
2622
2623
2624/**
2625 * Sets up the initial exception bitmap in the VMCS based on static conditions
2626 * (i.e. conditions that cannot ever change after starting the VM).
2627 *
2628 * @returns VBox status code.
2629 * @param pVM Pointer to the VM.
2630 * @param pVCpu Pointer to the VMCPU.
2631 */
2632static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2633{
2634 AssertPtr(pVM);
2635 AssertPtr(pVCpu);
2636
2637 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2638
2639 uint32_t u32XcptBitmap = 0;
2640
2641 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2642 if (!pVM->hm.s.fNestedPaging)
2643 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2644
2645 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2646 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2647 AssertRCReturn(rc, rc);
2648 return rc;
2649}
2650
2651
2652/**
2653 * Sets up the initial guest-state mask. The guest-state mask is consulted
2654 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2655 * for the nested virtualization case (as it would cause a VM-exit).
2656 *
2657 * @param pVCpu Pointer to the VMCPU.
2658 */
2659static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2660{
2661 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2662 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2663 return VINF_SUCCESS;
2664}
2665
2666
2667/**
2668 * Does per-VM VT-x initialization.
2669 *
2670 * @returns VBox status code.
2671 * @param pVM Pointer to the VM.
2672 */
2673VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2674{
2675 LogFlowFunc(("pVM=%p\n", pVM));
2676
2677 int rc = hmR0VmxStructsAlloc(pVM);
2678 if (RT_FAILURE(rc))
2679 {
2680 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2681 return rc;
2682 }
2683
2684 return VINF_SUCCESS;
2685}
2686
2687
2688/**
2689 * Does per-VM VT-x termination.
2690 *
2691 * @returns VBox status code.
2692 * @param pVM Pointer to the VM.
2693 */
2694VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2695{
2696 LogFlowFunc(("pVM=%p\n", pVM));
2697
2698#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2700 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2701#endif
2702 hmR0VmxStructsFree(pVM);
2703 return VINF_SUCCESS;
2704}
2705
2706
2707/**
2708 * Sets up the VM for execution under VT-x.
2709 * This function is only called once per-VM during initialization.
2710 *
2711 * @returns VBox status code.
2712 * @param pVM Pointer to the VM.
2713 */
2714VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2715{
2716 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2717 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2718
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721 /*
2722 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2723 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2724 */
2725 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2726 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2727 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2728 || !pVM->hm.s.vmx.pRealModeTSS))
2729 {
2730 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2731 return VERR_INTERNAL_ERROR;
2732 }
2733
2734#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2735 /*
2736 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2737 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2738 */
2739 if ( pVM->hm.s.fAllow64BitGuests
2740 && !HMVMX_IS_64BIT_HOST_MODE())
2741 {
2742 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2743 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2744 }
2745#endif
2746
2747 /* Initialize these always, see hmR3InitFinalizeR0().*/
2748 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2749 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2750
2751 /* Setup the tagged-TLB flush handlers. */
2752 int rc = hmR0VmxSetupTaggedTlb(pVM);
2753 if (RT_FAILURE(rc))
2754 {
2755 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2756 return rc;
2757 }
2758
2759 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2760 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2761#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2762 if ( HMVMX_IS_64BIT_HOST_MODE()
2763 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2764 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2765 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2766 {
2767 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2768 }
2769#endif
2770
2771 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2772 {
2773 PVMCPU pVCpu = &pVM->aCpus[i];
2774 AssertPtr(pVCpu);
2775 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2776
2777 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2778 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2779
2780 /* Set revision dword at the beginning of the VMCS structure. */
2781 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2782
2783 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2784 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2786 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2787
2788 /* Load this VMCS as the current VMCS. */
2789 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2791 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2792
2793 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2794 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2795 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2796
2797 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2799 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2800
2801 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2803 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2804
2805 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2807 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2808
2809 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2811 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2812
2813#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2814 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2815 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2816 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2817#endif
2818
2819 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2820 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2821 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2822 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2823
2824 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2825
2826 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2827 }
2828
2829 return VINF_SUCCESS;
2830}
2831
2832
2833/**
2834 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2835 * the VMCS.
2836 *
2837 * @returns VBox status code.
2838 * @param pVM Pointer to the VM.
2839 * @param pVCpu Pointer to the VMCPU.
2840 */
2841DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2842{
2843 NOREF(pVM); NOREF(pVCpu);
2844
2845 RTCCUINTREG uReg = ASMGetCR0();
2846 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847 AssertRCReturn(rc, rc);
2848
2849#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2850 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2851 if (HMVMX_IS_64BIT_HOST_MODE())
2852 {
2853 uint64_t uRegCR3 = HMR0Get64bitCR3();
2854 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2855 }
2856 else
2857#endif
2858 {
2859 uReg = ASMGetCR3();
2860 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2861 }
2862 AssertRCReturn(rc, rc);
2863
2864 uReg = ASMGetCR4();
2865 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2866 AssertRCReturn(rc, rc);
2867 return rc;
2868}
2869
2870
2871#if HC_ARCH_BITS == 64
2872/**
2873 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2874 * requirements. See hmR0VmxSaveHostSegmentRegs().
2875 */
2876# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2877 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2878 { \
2879 bool fValidSelector = true; \
2880 if ((selValue) & X86_SEL_LDT) \
2881 { \
2882 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2883 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2884 } \
2885 if (fValidSelector) \
2886 { \
2887 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2888 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2889 } \
2890 (selValue) = 0; \
2891 }
2892#endif
2893
2894
2895/**
2896 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2897 * the host-state area in the VMCS.
2898 *
2899 * @returns VBox status code.
2900 * @param pVM Pointer to the VM.
2901 * @param pVCpu Pointer to the VMCPU.
2902 */
2903DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2904{
2905 NOREF(pVM);
2906 int rc = VERR_INTERNAL_ERROR_5;
2907
2908#if HC_ARCH_BITS == 64
2909 /*
2910 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2911 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2912 */
2913 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2914 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2915#endif
2916
2917 /*
2918 * Host DS, ES, FS and GS segment registers.
2919 */
2920#if HC_ARCH_BITS == 64
2921 RTSEL uSelDS = ASMGetDS();
2922 RTSEL uSelES = ASMGetES();
2923 RTSEL uSelFS = ASMGetFS();
2924 RTSEL uSelGS = ASMGetGS();
2925#else
2926 RTSEL uSelDS = 0;
2927 RTSEL uSelES = 0;
2928 RTSEL uSelFS = 0;
2929 RTSEL uSelGS = 0;
2930#endif
2931
2932 /* Recalculate which host-state bits need to be manually restored. */
2933 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2934
2935 /*
2936 * Host CS and SS segment registers.
2937 */
2938#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2939 RTSEL uSelCS;
2940 RTSEL uSelSS;
2941 if (HMVMX_IS_64BIT_HOST_MODE())
2942 {
2943 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2944 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2945 }
2946 else
2947 {
2948 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2949 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2950 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2951 }
2952#else
2953 RTSEL uSelCS = ASMGetCS();
2954 RTSEL uSelSS = ASMGetSS();
2955#endif
2956
2957 /*
2958 * Host TR segment register.
2959 */
2960 RTSEL uSelTR = ASMGetTR();
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2965 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2966 */
2967 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2968 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2969 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2970 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2971# undef VMXLOCAL_ADJUST_HOST_SEG
2972#endif
2973
2974 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2975 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2976 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2977 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2978 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2979 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2980 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2981 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2982 Assert(uSelCS);
2983 Assert(uSelTR);
2984
2985 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2986#if 0
2987 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2988 Assert(uSelSS != 0);
2989#endif
2990
2991 /* Write these host selector fields into the host-state area in the VMCS. */
2992 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2994#if HC_ARCH_BITS == 64
2995 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2996 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2997 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2998 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2999#endif
3000 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
3001
3002 /*
3003 * Host GDTR and IDTR.
3004 */
3005 RTGDTR Gdtr;
3006 RT_ZERO(Gdtr);
3007#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3008 if (HMVMX_IS_64BIT_HOST_MODE())
3009 {
3010 X86XDTR64 Gdtr64;
3011 X86XDTR64 Idtr64;
3012 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
3013 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
3014 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
3015
3016 Gdtr.cbGdt = Gdtr64.cb;
3017 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
3018 }
3019 else
3020#endif
3021 {
3022 RTIDTR Idtr;
3023 ASMGetGDTR(&Gdtr);
3024 ASMGetIDTR(&Idtr);
3025 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
3026 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
3027
3028#if HC_ARCH_BITS == 64
3029 /*
3030 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3031 * maximum limit (0xffff) on every VM-exit.
3032 */
3033 if (Gdtr.cbGdt != 0xffff)
3034 {
3035 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3036 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3037 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3038 }
3039
3040 /*
3041 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3042 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3043 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3044 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3045 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3046 * hosts where we are pretty sure it won't cause trouble.
3047 */
3048# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3049 if (Idtr.cbIdt < 0x0fff)
3050# else
3051 if (Idtr.cbIdt != 0xffff)
3052# endif
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3055 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3056 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3057 }
3058#endif
3059 }
3060
3061 /*
3062 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3063 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3064 */
3065 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3066 {
3067 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3068 return VERR_VMX_INVALID_HOST_STATE;
3069 }
3070
3071 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3072#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3073 if (HMVMX_IS_64BIT_HOST_MODE())
3074 {
3075 /* We need the 64-bit TR base for hybrid darwin. */
3076 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3077 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3078 }
3079 else
3080#endif
3081 {
3082 uintptr_t uTRBase;
3083#if HC_ARCH_BITS == 64
3084 uTRBase = X86DESC64_BASE(pDesc);
3085
3086 /*
3087 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3088 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3089 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3090 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3091 *
3092 * [1] See Intel spec. 3.5 "System Descriptor Types".
3093 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3094 */
3095 Assert(pDesc->System.u4Type == 11);
3096 if ( pDesc->System.u16LimitLow != 0x67
3097 || pDesc->System.u4LimitHigh)
3098 {
3099 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3100 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3101
3102 /* Store the GDTR here as we need it while restoring TR. */
3103 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3104 }
3105#else
3106 uTRBase = X86DESC_BASE(pDesc);
3107#endif
3108 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3109 }
3110 AssertRCReturn(rc, rc);
3111
3112 /*
3113 * Host FS base and GS base.
3114 */
3115#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3116 if (HMVMX_IS_64BIT_HOST_MODE())
3117 {
3118 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3119 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3120 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3121 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3122
3123# if HC_ARCH_BITS == 64
3124 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3125 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3126 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3127 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3128 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3129# endif
3130 }
3131#endif
3132 return rc;
3133}
3134
3135
3136/**
3137 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3138 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3139 * the host after every successful VM-exit.
3140 *
3141 * @returns VBox status code.
3142 * @param pVM Pointer to the VM.
3143 * @param pVCpu Pointer to the VMCPU.
3144 *
3145 * @remarks No-long-jump zone!!!
3146 */
3147DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3148{
3149 NOREF(pVM);
3150
3151 AssertPtr(pVCpu);
3152 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3153
3154 int rc = VINF_SUCCESS;
3155#if HC_ARCH_BITS == 64
3156 if (pVM->hm.s.fAllow64BitGuests)
3157 hmR0VmxLazySaveHostMsrs(pVCpu);
3158#endif
3159
3160 /*
3161 * Host Sysenter MSRs.
3162 */
3163 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3164 AssertRCReturn(rc, rc);
3165#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3166 if (HMVMX_IS_64BIT_HOST_MODE())
3167 {
3168 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3169 AssertRCReturn(rc, rc);
3170 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3171 }
3172 else
3173 {
3174 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3175 AssertRCReturn(rc, rc);
3176 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3177 }
3178#elif HC_ARCH_BITS == 32
3179 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3180 AssertRCReturn(rc, rc);
3181 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3182#else
3183 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3184 AssertRCReturn(rc, rc);
3185 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3186#endif
3187 AssertRCReturn(rc, rc);
3188
3189 /*
3190 * Host EFER MSR.
3191 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3192 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3193 */
3194 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3195 {
3196 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3197 AssertRCReturn(rc, rc);
3198 }
3199
3200 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3201 * hmR0VmxLoadGuestExitCtls() !! */
3202
3203 return rc;
3204}
3205
3206
3207/**
3208 * Figures out if we need to swap the EFER MSR which is
3209 * particularly expensive.
3210 *
3211 * We check all relevant bits. For now, that's everything
3212 * besides LMA/LME, as these two bits are handled by VM-entry,
3213 * see hmR0VmxLoadGuestExitCtls() and
3214 * hmR0VMxLoadGuestEntryCtls().
3215 *
3216 * @returns true if we need to load guest EFER, false otherwise.
3217 * @param pVCpu Pointer to the VMCPU.
3218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3219 * out-of-sync. Make sure to update the required fields
3220 * before using them.
3221 *
3222 * @remarks Requires EFER, CR4.
3223 * @remarks No-long-jump zone!!!
3224 */
3225static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3226{
3227#ifdef HMVMX_ALWAYS_SWAP_EFER
3228 return true;
3229#endif
3230
3231#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3232 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3233 if (CPUMIsGuestInLongMode(pVCpu))
3234 return false;
3235#endif
3236
3237 PVM pVM = pVCpu->CTX_SUFF(pVM);
3238 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3239 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3240
3241 /*
3242 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3243 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3244 */
3245 if ( CPUMIsGuestInLongMode(pVCpu)
3246 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3247 {
3248 return true;
3249 }
3250
3251 /*
3252 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it .
3253 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3254 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3255 */
3256 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3257 && (pMixedCtx->cr0 & X86_CR0_PG)
3258 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3259 {
3260 /* Assert that host is PAE capable. */
3261 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3262 return true;
3263 }
3264
3265 /** @todo Check the latest Intel spec. for any other bits,
3266 * like SMEP/SMAP? */
3267 return false;
3268}
3269
3270
3271/**
3272 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3273 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3274 * controls".
3275 *
3276 * @returns VBox status code.
3277 * @param pVCpu Pointer to the VMCPU.
3278 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3279 * out-of-sync. Make sure to update the required fields
3280 * before using them.
3281 *
3282 * @remarks Requires EFER.
3283 * @remarks No-long-jump zone!!!
3284 */
3285DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3286{
3287 int rc = VINF_SUCCESS;
3288 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3289 {
3290 PVM pVM = pVCpu->CTX_SUFF(pVM);
3291 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3292 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3293
3294 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3295 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3296
3297 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3298 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3299 {
3300 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3301 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3302 }
3303 else
3304 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3305
3306 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3307 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3308 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3309 {
3310 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3311 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3312 }
3313
3314 /*
3315 * The following should -not- be set (since we're not in SMM mode):
3316 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3317 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3318 */
3319
3320 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3321 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3322
3323 if ((val & zap) != val)
3324 {
3325 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3326 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3327 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3328 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3329 }
3330
3331 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3332 AssertRCReturn(rc, rc);
3333
3334 /* Update VCPU with the currently set VM-exit controls. */
3335 pVCpu->hm.s.vmx.u32EntryCtls = val;
3336 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3337 }
3338 return rc;
3339}
3340
3341
3342/**
3343 * Sets up the VM-exit controls in the VMCS.
3344 *
3345 * @returns VBox status code.
3346 * @param pVM Pointer to the VM.
3347 * @param pVCpu Pointer to the VMCPU.
3348 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3349 * out-of-sync. Make sure to update the required fields
3350 * before using them.
3351 *
3352 * @remarks Requires EFER.
3353 */
3354DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3355{
3356 NOREF(pMixedCtx);
3357
3358 int rc = VINF_SUCCESS;
3359 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3360 {
3361 PVM pVM = pVCpu->CTX_SUFF(pVM);
3362 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3363 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3364
3365 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3366 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3367
3368 /*
3369 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3370 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3371 */
3372#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3373 if (HMVMX_IS_64BIT_HOST_MODE())
3374 {
3375 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3376 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3377 }
3378 else
3379 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3380#else
3381 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3382 {
3383 /* The switcher returns to long mode, EFER is managed by the switcher. */
3384 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3385 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3386 }
3387 else
3388 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3389#endif /* HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
3390
3391 /* If the newer VMCS fields for managing EFER exists, use it. */
3392 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3393 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3394 {
3395 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3396 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3398 }
3399
3400 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3401 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3402
3403 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3404 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3405 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3406
3407 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3408 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3409
3410 if ((val & zap) != val)
3411 {
3412 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3413 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3414 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3415 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3416 }
3417
3418 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3419 AssertRCReturn(rc, rc);
3420
3421 /* Update VCPU with the currently set VM-exit controls. */
3422 pVCpu->hm.s.vmx.u32ExitCtls = val;
3423 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3424 }
3425 return rc;
3426}
3427
3428
3429/**
3430 * Loads the guest APIC and related state.
3431 *
3432 * @returns VBox status code.
3433 * @param pVM Pointer to the VM.
3434 * @param pVCpu Pointer to the VMCPU.
3435 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3436 * out-of-sync. Make sure to update the required fields
3437 * before using them.
3438 */
3439DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3440{
3441 NOREF(pMixedCtx);
3442
3443 int rc = VINF_SUCCESS;
3444 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3445 {
3446 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3448 {
3449 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3450
3451 bool fPendingIntr = false;
3452 uint8_t u8Tpr = 0;
3453 uint8_t u8PendingIntr = 0;
3454 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3455 AssertRCReturn(rc, rc);
3456
3457 /*
3458 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3459 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3460 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3461 * the interrupt when we VM-exit for other reasons.
3462 */
3463 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3464 uint32_t u32TprThreshold = 0;
3465 if (fPendingIntr)
3466 {
3467 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3468 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3469 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3470 if (u8PendingPriority <= u8TprPriority)
3471 u32TprThreshold = u8PendingPriority;
3472 else
3473 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3474 }
3475 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3476
3477 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3478 AssertRCReturn(rc, rc);
3479 }
3480
3481 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3482 }
3483 return rc;
3484}
3485
3486
3487/**
3488 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3489 *
3490 * @returns Guest's interruptibility-state.
3491 * @param pVCpu Pointer to the VMCPU.
3492 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3493 * out-of-sync. Make sure to update the required fields
3494 * before using them.
3495 *
3496 * @remarks No-long-jump zone!!!
3497 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3498 */
3499DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3500{
3501 /*
3502 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3503 * inhibit interrupts or clear any existing interrupt-inhibition.
3504 */
3505 uint32_t uIntrState = 0;
3506 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3507 {
3508 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3509 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3510 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3511 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3512 {
3513 /*
3514 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3515 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3516 */
3517 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3518 }
3519 else if (pMixedCtx->eflags.Bits.u1IF)
3520 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3521 else
3522 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3523 }
3524
3525 /*
3526 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3527 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3528 * setting this would block host-NMIs and IRET will not clear the blocking.
3529 *
3530 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3531 */
3532 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3533 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3534 {
3535 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3536 }
3537
3538 return uIntrState;
3539}
3540
3541
3542/**
3543 * Loads the guest's interruptibility-state into the guest-state area in the
3544 * VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu Pointer to the VMCPU.
3548 * @param uIntrState The interruptibility-state to set.
3549 */
3550static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3551{
3552 NOREF(pVCpu);
3553 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3554 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3555 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3556 AssertRCReturn(rc, rc);
3557 return rc;
3558}
3559
3560
3561/**
3562 * Loads the guest's RIP into the guest-state area in the VMCS.
3563 *
3564 * @returns VBox status code.
3565 * @param pVCpu Pointer to the VMCPU.
3566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3567 * out-of-sync. Make sure to update the required fields
3568 * before using them.
3569 *
3570 * @remarks No-long-jump zone!!!
3571 */
3572static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3573{
3574 int rc = VINF_SUCCESS;
3575 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3576 {
3577 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3578 AssertRCReturn(rc, rc);
3579
3580 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3581 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3582 HMCPU_CF_VALUE(pVCpu)));
3583 }
3584 return rc;
3585}
3586
3587
3588/**
3589 * Loads the guest's RSP into the guest-state area in the VMCS.
3590 *
3591 * @returns VBox status code.
3592 * @param pVCpu Pointer to the VMCPU.
3593 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3594 * out-of-sync. Make sure to update the required fields
3595 * before using them.
3596 *
3597 * @remarks No-long-jump zone!!!
3598 */
3599static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3600{
3601 int rc = VINF_SUCCESS;
3602 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3603 {
3604 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3605 AssertRCReturn(rc, rc);
3606
3607 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3608 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3609 }
3610 return rc;
3611}
3612
3613
3614/**
3615 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3616 *
3617 * @returns VBox status code.
3618 * @param pVCpu Pointer to the VMCPU.
3619 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3620 * out-of-sync. Make sure to update the required fields
3621 * before using them.
3622 *
3623 * @remarks No-long-jump zone!!!
3624 */
3625static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3626{
3627 int rc = VINF_SUCCESS;
3628 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3629 {
3630 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3631 Let us assert it as such and use 32-bit VMWRITE. */
3632 Assert(!(pMixedCtx->rflags.u64 >> 32));
3633 X86EFLAGS Eflags = pMixedCtx->eflags;
3634 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3635 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3636
3637 /*
3638 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3639 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3640 */
3641 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3642 {
3643 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3644 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3645 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3646 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3647 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3648 }
3649
3650 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3651 AssertRCReturn(rc, rc);
3652
3653 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3654 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu Pointer to the VMCPU.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3674 AssertRCReturn(rc, rc);
3675 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3676 AssertRCReturn(rc, rc);
3677 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3678 AssertRCReturn(rc, rc);
3679 return rc;
3680}
3681
3682
3683/**
3684 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3685 * CR0 is partially shared with the host and we have to consider the FPU bits.
3686 *
3687 * @returns VBox status code.
3688 * @param pVM Pointer to the VM.
3689 * @param pVCpu Pointer to the VMCPU.
3690 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3691 * out-of-sync. Make sure to update the required fields
3692 * before using them.
3693 *
3694 * @remarks No-long-jump zone!!!
3695 */
3696static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3697{
3698 /*
3699 * Guest CR0.
3700 * Guest FPU.
3701 */
3702 int rc = VINF_SUCCESS;
3703 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3704 {
3705 Assert(!(pMixedCtx->cr0 >> 32));
3706 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3707 PVM pVM = pVCpu->CTX_SUFF(pVM);
3708
3709 /* The guest's view (read access) of its CR0 is unblemished. */
3710 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3711 AssertRCReturn(rc, rc);
3712 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3713
3714 /* Setup VT-x's view of the guest CR0. */
3715 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3716 if (pVM->hm.s.fNestedPaging)
3717 {
3718 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3719 {
3720 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3721 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3723 }
3724 else
3725 {
3726 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3727 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3728 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3729 }
3730
3731 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3732 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3733 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3734
3735 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3736 AssertRCReturn(rc, rc);
3737 }
3738 else
3739 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3740
3741 /*
3742 * Guest FPU bits.
3743 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3744 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3745 */
3746 u32GuestCR0 |= X86_CR0_NE;
3747 bool fInterceptNM = false;
3748 if (CPUMIsGuestFPUStateActive(pVCpu))
3749 {
3750 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3751 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3752 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3753 }
3754 else
3755 {
3756 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3757 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3758 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3759 }
3760
3761 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3762 bool fInterceptMF = false;
3763 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3764 fInterceptMF = true;
3765
3766 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3767 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3768 {
3769 Assert(PDMVmmDevHeapIsEnabled(pVM));
3770 Assert(pVM->hm.s.vmx.pRealModeTSS);
3771 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3772 fInterceptNM = true;
3773 fInterceptMF = true;
3774 }
3775 else
3776 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3777
3778 if (fInterceptNM)
3779 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3780 else
3781 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3782
3783 if (fInterceptMF)
3784 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3785 else
3786 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3787
3788 /* Additional intercepts for debugging, define these yourself explicitly. */
3789#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3790 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3791 | RT_BIT(X86_XCPT_BP)
3792 | RT_BIT(X86_XCPT_DB)
3793 | RT_BIT(X86_XCPT_DE)
3794 | RT_BIT(X86_XCPT_NM)
3795 | RT_BIT(X86_XCPT_TS)
3796 | RT_BIT(X86_XCPT_UD)
3797 | RT_BIT(X86_XCPT_NP)
3798 | RT_BIT(X86_XCPT_SS)
3799 | RT_BIT(X86_XCPT_GP)
3800 | RT_BIT(X86_XCPT_PF)
3801 | RT_BIT(X86_XCPT_MF)
3802 ;
3803#elif defined(HMVMX_ALWAYS_TRAP_PF)
3804 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3805#endif
3806
3807 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3808
3809 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3810 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3811 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3812 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3813 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3814 else
3815 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3816
3817 u32GuestCR0 |= uSetCR0;
3818 u32GuestCR0 &= uZapCR0;
3819 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3820
3821 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3822 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3823 AssertRCReturn(rc, rc);
3824 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3825 AssertRCReturn(rc, rc);
3826 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3827 uZapCR0));
3828
3829 /*
3830 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3831 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3832 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3833 */
3834 uint32_t u32CR0Mask = 0;
3835 u32CR0Mask = X86_CR0_PE
3836 | X86_CR0_NE
3837 | X86_CR0_WP
3838 | X86_CR0_PG
3839 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3840 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3841 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3842
3843 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3844 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3845 * and @bugref{6944}. */
3846#if 0
3847 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3848 u32CR0Mask &= ~X86_CR0_PE;
3849#endif
3850 if (pVM->hm.s.fNestedPaging)
3851 u32CR0Mask &= ~X86_CR0_WP;
3852
3853 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3854 if (fInterceptNM)
3855 {
3856 u32CR0Mask |= X86_CR0_TS
3857 | X86_CR0_MP;
3858 }
3859
3860 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3861 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3862 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3863 AssertRCReturn(rc, rc);
3864 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3865
3866 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3867 }
3868 return rc;
3869}
3870
3871
3872/**
3873 * Loads the guest control registers (CR3, CR4) into the guest-state area
3874 * in the VMCS.
3875 *
3876 * @returns VBox status code.
3877 * @param pVM Pointer to the VM.
3878 * @param pVCpu Pointer to the VMCPU.
3879 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3880 * out-of-sync. Make sure to update the required fields
3881 * before using them.
3882 *
3883 * @remarks No-long-jump zone!!!
3884 */
3885static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3886{
3887 int rc = VINF_SUCCESS;
3888 PVM pVM = pVCpu->CTX_SUFF(pVM);
3889
3890 /*
3891 * Guest CR2.
3892 * It's always loaded in the assembler code. Nothing to do here.
3893 */
3894
3895 /*
3896 * Guest CR3.
3897 */
3898 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3899 {
3900 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3901 if (pVM->hm.s.fNestedPaging)
3902 {
3903 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3904
3905 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3906 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3907 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3908 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3909
3910 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3911 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3912 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3913
3914 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3915 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3916 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3917 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3918
3919 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3920 AssertRCReturn(rc, rc);
3921 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3922
3923 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3924 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3925 {
3926 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3927 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3928 {
3929 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3930 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3931 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3932 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3933 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3934 }
3935
3936 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3937 have Unrestricted Execution to handle the guest when it's not using paging. */
3938 GCPhysGuestCR3 = pMixedCtx->cr3;
3939 }
3940 else
3941 {
3942 /*
3943 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3944 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3945 * EPT takes care of translating it to host-physical addresses.
3946 */
3947 RTGCPHYS GCPhys;
3948 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3949 Assert(PDMVmmDevHeapIsEnabled(pVM));
3950
3951 /* We obtain it here every time as the guest could have relocated this PCI region. */
3952 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3953 AssertRCReturn(rc, rc);
3954
3955 GCPhysGuestCR3 = GCPhys;
3956 }
3957
3958 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
3959 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3960 }
3961 else
3962 {
3963 /* Non-nested paging case, just use the hypervisor's CR3. */
3964 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3965
3966 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
3967 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3968 }
3969 AssertRCReturn(rc, rc);
3970
3971 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3972 }
3973
3974 /*
3975 * Guest CR4.
3976 */
3977 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3978 {
3979 Assert(!(pMixedCtx->cr4 >> 32));
3980 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3981
3982 /* The guest's view of its CR4 is unblemished. */
3983 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3984 AssertRCReturn(rc, rc);
3985 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
3986
3987 /* Setup VT-x's view of the guest CR4. */
3988 /*
3989 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3990 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3991 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3992 */
3993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3994 {
3995 Assert(pVM->hm.s.vmx.pRealModeTSS);
3996 Assert(PDMVmmDevHeapIsEnabled(pVM));
3997 u32GuestCR4 &= ~X86_CR4_VME;
3998 }
3999
4000 if (pVM->hm.s.fNestedPaging)
4001 {
4002 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4003 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4004 {
4005 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4006 u32GuestCR4 |= X86_CR4_PSE;
4007 /* Our identity mapping is a 32-bit page directory. */
4008 u32GuestCR4 &= ~X86_CR4_PAE;
4009 }
4010 /* else use guest CR4.*/
4011 }
4012 else
4013 {
4014 /*
4015 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4016 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4017 */
4018 switch (pVCpu->hm.s.enmShadowMode)
4019 {
4020 case PGMMODE_REAL: /* Real-mode. */
4021 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4022 case PGMMODE_32_BIT: /* 32-bit paging. */
4023 {
4024 u32GuestCR4 &= ~X86_CR4_PAE;
4025 break;
4026 }
4027
4028 case PGMMODE_PAE: /* PAE paging. */
4029 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4030 {
4031 u32GuestCR4 |= X86_CR4_PAE;
4032 break;
4033 }
4034
4035 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4036 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4037#ifdef VBOX_ENABLE_64_BITS_GUESTS
4038 break;
4039#endif
4040 default:
4041 AssertFailed();
4042 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4043 }
4044 }
4045
4046 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4047 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4049 u32GuestCR4 |= uSetCR4;
4050 u32GuestCR4 &= uZapCR4;
4051
4052 /* Write VT-x's view of the guest CR4 into the VMCS. */
4053 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4054 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4055 AssertRCReturn(rc, rc);
4056
4057 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4058 uint32_t u32CR4Mask = 0;
4059 u32CR4Mask = X86_CR4_VME
4060 | X86_CR4_PAE
4061 | X86_CR4_PGE
4062 | X86_CR4_PSE
4063 | X86_CR4_VMXE;
4064 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4065 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4066 AssertRCReturn(rc, rc);
4067
4068 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4069 }
4070 return rc;
4071}
4072
4073
4074/**
4075 * Loads the guest debug registers into the guest-state area in the VMCS.
4076 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4077 *
4078 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4079 *
4080 * @returns VBox status code.
4081 * @param pVCpu Pointer to the VMCPU.
4082 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4083 * out-of-sync. Make sure to update the required fields
4084 * before using them.
4085 *
4086 * @remarks No-long-jump zone!!!
4087 */
4088static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4089{
4090 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4091 return VINF_SUCCESS;
4092
4093#ifdef VBOX_STRICT
4094 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4095 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4096 {
4097 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4098 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4099 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4100 }
4101#endif
4102
4103 int rc;
4104 PVM pVM = pVCpu->CTX_SUFF(pVM);
4105 bool fInterceptDB = false;
4106 bool fInterceptMovDRx = false;
4107 if ( pVCpu->hm.s.fSingleInstruction
4108 || DBGFIsStepping(pVCpu))
4109 {
4110 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4111 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4112 {
4113 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4114 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4115 AssertRCReturn(rc, rc);
4116 Assert(fInterceptDB == false);
4117 }
4118 else
4119 {
4120 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4121 pVCpu->hm.s.fClearTrapFlag = true;
4122 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4123 fInterceptDB = true;
4124 }
4125 }
4126
4127 if ( fInterceptDB
4128 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4129 {
4130 /*
4131 * Use the combined guest and host DRx values found in the hypervisor
4132 * register set because the debugger has breakpoints active or someone
4133 * is single stepping on the host side without a monitor trap flag.
4134 *
4135 * Note! DBGF expects a clean DR6 state before executing guest code.
4136 */
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4138 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4139 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4144 }
4145 else
4146#endif
4147 if (!CPUMIsHyperDebugStateActive(pVCpu))
4148 {
4149 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4150 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4151 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4152 }
4153
4154 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4155 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4156 AssertRCReturn(rc, rc);
4157
4158 pVCpu->hm.s.fUsingHyperDR7 = true;
4159 fInterceptDB = true;
4160 fInterceptMovDRx = true;
4161 }
4162 else
4163 {
4164 /*
4165 * If the guest has enabled debug registers, we need to load them prior to
4166 * executing guest code so they'll trigger at the right time.
4167 */
4168 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4169 {
4170#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4171 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4172 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4173 {
4174 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4175 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4176 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4177 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4178 }
4179 else
4180#endif
4181 if (!CPUMIsGuestDebugStateActive(pVCpu))
4182 {
4183 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4184 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4185 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4186 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4187 }
4188 Assert(!fInterceptDB);
4189 Assert(!fInterceptMovDRx);
4190 }
4191 /*
4192 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4193 * must intercept #DB in order to maintain a correct DR6 guest value.
4194 */
4195#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4196 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4197 && !CPUMIsGuestDebugStateActive(pVCpu))
4198#else
4199 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4200#endif
4201 {
4202 fInterceptMovDRx = true;
4203 fInterceptDB = true;
4204 }
4205
4206 /* Update guest DR7. */
4207 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4208 AssertRCReturn(rc, rc);
4209
4210 pVCpu->hm.s.fUsingHyperDR7 = false;
4211 }
4212
4213 /*
4214 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4215 */
4216 if (fInterceptDB)
4217 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4218 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4219 {
4220#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4221 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4222#endif
4223 }
4224 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4225 AssertRCReturn(rc, rc);
4226
4227 /*
4228 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4229 */
4230 if (fInterceptMovDRx)
4231 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4232 else
4233 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4234 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4235 AssertRCReturn(rc, rc);
4236
4237 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4238 return VINF_SUCCESS;
4239}
4240
4241
4242#ifdef VBOX_STRICT
4243/**
4244 * Strict function to validate segment registers.
4245 *
4246 * @remarks ASSUMES CR0 is up to date.
4247 */
4248static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4249{
4250 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4251 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4252 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4253 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4254 && ( !CPUMIsGuestInRealModeEx(pCtx)
4255 && !CPUMIsGuestInV86ModeEx(pCtx)))
4256 {
4257 /* Protected mode checks */
4258 /* CS */
4259 Assert(pCtx->cs.Attr.n.u1Present);
4260 Assert(!(pCtx->cs.Attr.u & 0xf00));
4261 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->cs.Attr.n.u1Granularity));
4264 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4265 || (pCtx->cs.Attr.n.u1Granularity));
4266 /* CS cannot be loaded with NULL in protected mode. */
4267 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4268 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4269 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4270 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4271 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4272 else
4273 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4274 /* SS */
4275 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4276 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4277 if ( !(pCtx->cr0 & X86_CR0_PE)
4278 || pCtx->cs.Attr.n.u4Type == 3)
4279 {
4280 Assert(!pCtx->ss.Attr.n.u2Dpl);
4281 }
4282 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4286 Assert(pCtx->ss.Attr.n.u1Present);
4287 Assert(!(pCtx->ss.Attr.u & 0xf00));
4288 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ss.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4292 || (pCtx->ss.Attr.n.u1Granularity));
4293 }
4294 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4295 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->ds.Attr.n.u1Present);
4299 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->ds.Attr.u & 0xf00));
4301 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->ds.Attr.n.u1Granularity));
4304 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4305 || (pCtx->ds.Attr.n.u1Granularity));
4306 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->es.Attr.n.u1Present);
4313 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->es.Attr.u & 0xf00));
4315 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->es.Attr.n.u1Granularity));
4318 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4319 || (pCtx->es.Attr.n.u1Granularity));
4320 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->fs.Attr.n.u1Present);
4327 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->fs.Attr.u & 0xf00));
4329 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->fs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4333 || (pCtx->fs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4338 {
4339 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4340 Assert(pCtx->gs.Attr.n.u1Present);
4341 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4342 Assert(!(pCtx->gs.Attr.u & 0xf00));
4343 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4344 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4345 || !(pCtx->gs.Attr.n.u1Granularity));
4346 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4347 || (pCtx->gs.Attr.n.u1Granularity));
4348 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4349 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4350 }
4351 /* 64-bit capable CPUs. */
4352# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4353 if (HMVMX_IS_64BIT_HOST_MODE())
4354 {
4355 Assert(!(pCtx->cs.u64Base >> 32));
4356 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4357 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4358 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4359 }
4360# endif
4361 }
4362 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4363 || ( CPUMIsGuestInRealModeEx(pCtx)
4364 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4365 {
4366 /* Real and v86 mode checks. */
4367 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4368 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4369 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4370 {
4371 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4372 }
4373 else
4374 {
4375 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4376 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4377 }
4378
4379 /* CS */
4380 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4381 Assert(pCtx->cs.u32Limit == 0xffff);
4382 Assert(u32CSAttr == 0xf3);
4383 /* SS */
4384 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4385 Assert(pCtx->ss.u32Limit == 0xffff);
4386 Assert(u32SSAttr == 0xf3);
4387 /* DS */
4388 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4389 Assert(pCtx->ds.u32Limit == 0xffff);
4390 Assert(u32DSAttr == 0xf3);
4391 /* ES */
4392 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4393 Assert(pCtx->es.u32Limit == 0xffff);
4394 Assert(u32ESAttr == 0xf3);
4395 /* FS */
4396 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4397 Assert(pCtx->fs.u32Limit == 0xffff);
4398 Assert(u32FSAttr == 0xf3);
4399 /* GS */
4400 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4401 Assert(pCtx->gs.u32Limit == 0xffff);
4402 Assert(u32GSAttr == 0xf3);
4403 /* 64-bit capable CPUs. */
4404# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4405 if (HMVMX_IS_64BIT_HOST_MODE())
4406 {
4407 Assert(!(pCtx->cs.u64Base >> 32));
4408 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4409 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4410 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4411 }
4412# endif
4413 }
4414}
4415#endif /* VBOX_STRICT */
4416
4417
4418/**
4419 * Writes a guest segment register into the guest-state area in the VMCS.
4420 *
4421 * @returns VBox status code.
4422 * @param pVCpu Pointer to the VMCPU.
4423 * @param idxSel Index of the selector in the VMCS.
4424 * @param idxLimit Index of the segment limit in the VMCS.
4425 * @param idxBase Index of the segment base in the VMCS.
4426 * @param idxAccess Index of the access rights of the segment in the VMCS.
4427 * @param pSelReg Pointer to the segment selector.
4428 *
4429 * @remarks No-long-jump zone!!!
4430 */
4431static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4432 uint32_t idxAccess, PCPUMSELREG pSelReg)
4433{
4434 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4435 AssertRCReturn(rc, rc);
4436 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4437 AssertRCReturn(rc, rc);
4438 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4439 AssertRCReturn(rc, rc);
4440
4441 uint32_t u32Access = pSelReg->Attr.u;
4442 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4443 {
4444 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4445 u32Access = 0xf3;
4446 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4447 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4448 }
4449 else
4450 {
4451 /*
4452 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4453 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4454 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4455 * loaded in protected-mode have their attribute as 0.
4456 */
4457 if (!u32Access)
4458 u32Access = X86DESCATTR_UNUSABLE;
4459 }
4460
4461 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4462 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4463 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4464
4465 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4466 AssertRCReturn(rc, rc);
4467 return rc;
4468}
4469
4470
4471/**
4472 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4473 * into the guest-state area in the VMCS.
4474 *
4475 * @returns VBox status code.
4476 * @param pVM Pointer to the VM.
4477 * @param pVCPU Pointer to the VMCPU.
4478 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4479 * out-of-sync. Make sure to update the required fields
4480 * before using them.
4481 *
4482 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4483 * @remarks No-long-jump zone!!!
4484 */
4485static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4486{
4487 int rc = VERR_INTERNAL_ERROR_5;
4488 PVM pVM = pVCpu->CTX_SUFF(pVM);
4489
4490 /*
4491 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4492 */
4493 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4494 {
4495 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4496 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4497 {
4498 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4499 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4500 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4501 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4502 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4503 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4504 }
4505
4506#ifdef VBOX_WITH_REM
4507 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4508 {
4509 Assert(pVM->hm.s.vmx.pRealModeTSS);
4510 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4511 if ( pVCpu->hm.s.vmx.fWasInRealMode
4512 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4513 {
4514 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4515 in real-mode (e.g. OpenBSD 4.0) */
4516 REMFlushTBs(pVM);
4517 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4518 pVCpu->hm.s.vmx.fWasInRealMode = false;
4519 }
4520 }
4521#endif
4522 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4523 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4524 AssertRCReturn(rc, rc);
4525 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4526 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4527 AssertRCReturn(rc, rc);
4528 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4529 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4530 AssertRCReturn(rc, rc);
4531 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4532 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4533 AssertRCReturn(rc, rc);
4534 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4535 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4536 AssertRCReturn(rc, rc);
4537 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4538 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4539 AssertRCReturn(rc, rc);
4540
4541#ifdef VBOX_STRICT
4542 /* Validate. */
4543 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4544#endif
4545
4546 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4547 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4548 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4549 }
4550
4551 /*
4552 * Guest TR.
4553 */
4554 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4555 {
4556 /*
4557 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4558 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4559 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4560 */
4561 uint16_t u16Sel = 0;
4562 uint32_t u32Limit = 0;
4563 uint64_t u64Base = 0;
4564 uint32_t u32AccessRights = 0;
4565
4566 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4567 {
4568 u16Sel = pMixedCtx->tr.Sel;
4569 u32Limit = pMixedCtx->tr.u32Limit;
4570 u64Base = pMixedCtx->tr.u64Base;
4571 u32AccessRights = pMixedCtx->tr.Attr.u;
4572 }
4573 else
4574 {
4575 Assert(pVM->hm.s.vmx.pRealModeTSS);
4576 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4577
4578 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4579 RTGCPHYS GCPhys;
4580 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4581 AssertRCReturn(rc, rc);
4582
4583 X86DESCATTR DescAttr;
4584 DescAttr.u = 0;
4585 DescAttr.n.u1Present = 1;
4586 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4587
4588 u16Sel = 0;
4589 u32Limit = HM_VTX_TSS_SIZE;
4590 u64Base = GCPhys; /* in real-mode phys = virt. */
4591 u32AccessRights = DescAttr.u;
4592 }
4593
4594 /* Validate. */
4595 Assert(!(u16Sel & RT_BIT(2)));
4596 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4597 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4598 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4599 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4600 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4601 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4602 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4603 Assert( (u32Limit & 0xfff) == 0xfff
4604 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4605 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4606 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4607
4608 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4609 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4610 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4611 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4612
4613 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4614 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4615 }
4616
4617 /*
4618 * Guest GDTR.
4619 */
4620 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4621 {
4622 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4623 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4624
4625 /* Validate. */
4626 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4627
4628 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4629 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4630 }
4631
4632 /*
4633 * Guest LDTR.
4634 */
4635 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4636 {
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pMixedCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pMixedCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4645 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4646 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4647 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4661 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4665 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4672 {
4673 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4674 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4675
4676 /* Validate. */
4677 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4678
4679 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4680 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4681 }
4682
4683 return VINF_SUCCESS;
4684}
4685
4686
4687/**
4688 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4689 * areas. These MSRs will automatically be loaded to the host CPU on every
4690 * successful VM entry and stored from the host CPU on every successful VM-exit.
4691 *
4692 * This also creates/updates MSR slots for the host MSRs. The actual host
4693 * MSR values are -not- updated here for performance reasons. See
4694 * hmR0VmxSaveHostMsrs().
4695 *
4696 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4697 *
4698 * @returns VBox status code.
4699 * @param pVCpu Pointer to the VMCPU.
4700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4701 * out-of-sync. Make sure to update the required fields
4702 * before using them.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 */
4714 PVM pVM = pVCpu->CTX_SUFF(pVM);
4715 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4716 {
4717 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4718#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4722 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4723 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4724 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4725# ifdef DEBUG
4726 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4727 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4728 {
4729 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4730 pMsr->u64Value));
4731 }
4732# endif
4733 }
4734#endif
4735 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4736 }
4737
4738 /*
4739 * Guest Sysenter MSRs.
4740 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4741 * VM-exits on WRMSRs for these MSRs.
4742 */
4743 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4744 {
4745 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4746 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4747 }
4748
4749 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4750 {
4751 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4752 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4753 }
4754
4755 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4758 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4759 }
4760
4761 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4762 {
4763 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4764 {
4765 /*
4766 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4767 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4768 */
4769 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4770 {
4771 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4772 AssertRCReturn(rc,rc);
4773 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4774 }
4775 else
4776 {
4777 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */);
4778 /* We need to intercept reads too, see @bugref{7386} comment #16. */
4779 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4780 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4781 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4782 }
4783 }
4784 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4785 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4786 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4787 }
4788
4789 return VINF_SUCCESS;
4790}
4791
4792
4793/**
4794 * Loads the guest activity state into the guest-state area in the VMCS.
4795 *
4796 * @returns VBox status code.
4797 * @param pVCpu Pointer to the VMCPU.
4798 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4799 * out-of-sync. Make sure to update the required fields
4800 * before using them.
4801 *
4802 * @remarks No-long-jump zone!!!
4803 */
4804static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4805{
4806 NOREF(pCtx);
4807 /** @todo See if we can make use of other states, e.g.
4808 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4809 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4810 {
4811 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4812 AssertRCReturn(rc, rc);
4813
4814 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4815 }
4816 return VINF_SUCCESS;
4817}
4818
4819
4820/**
4821 * Sets up the appropriate function to run guest code.
4822 *
4823 * @returns VBox status code.
4824 * @param pVCpu Pointer to the VMCPU.
4825 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4826 * out-of-sync. Make sure to update the required fields
4827 * before using them.
4828 *
4829 * @remarks No-long-jump zone!!!
4830 */
4831static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4832{
4833 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4834 {
4835#ifndef VBOX_ENABLE_64_BITS_GUESTS
4836 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4837#endif
4838 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4839#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4840 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4841 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4842 {
4843 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4844 {
4845 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4846 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4847 | HM_CHANGED_VMX_EXIT_CTLS
4848 | HM_CHANGED_VMX_ENTRY_CTLS
4849 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4850 }
4851 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4852 }
4853#else
4854 /* 64-bit host or hybrid host. */
4855 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4856#endif
4857 }
4858 else
4859 {
4860 /* Guest is not in long mode, use the 32-bit handler. */
4861#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4862 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4863 {
4864 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4865 {
4866 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4867 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4868 | HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4871 }
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4873 }
4874#else
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876#endif
4877 }
4878 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4879 return VINF_SUCCESS;
4880}
4881
4882
4883/**
4884 * Wrapper for running the guest code in VT-x.
4885 *
4886 * @returns VBox strict status code.
4887 * @param pVM Pointer to the VM.
4888 * @param pVCpu Pointer to the VMCPU.
4889 * @param pCtx Pointer to the guest-CPU context.
4890 *
4891 * @remarks No-long-jump zone!!!
4892 */
4893DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4894{
4895 /*
4896 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4897 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4898 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4899 */
4900 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4901 /** @todo Add stats for resume vs launch. */
4902#ifdef VBOX_WITH_KERNEL_USING_XMM
4903 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4904#else
4905 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4906#endif
4907}
4908
4909
4910/**
4911 * Reports world-switch error and dumps some useful debug info.
4912 *
4913 * @param pVM Pointer to the VM.
4914 * @param pVCpu Pointer to the VMCPU.
4915 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4916 * @param pCtx Pointer to the guest-CPU context.
4917 * @param pVmxTransient Pointer to the VMX transient structure (only
4918 * exitReason updated).
4919 */
4920static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4921{
4922 Assert(pVM);
4923 Assert(pVCpu);
4924 Assert(pCtx);
4925 Assert(pVmxTransient);
4926 HMVMX_ASSERT_PREEMPT_SAFE();
4927
4928 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4929 switch (rcVMRun)
4930 {
4931 case VERR_VMX_INVALID_VMXON_PTR:
4932 AssertFailed();
4933 break;
4934 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4935 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4936 {
4937 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4938 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4939 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4940 AssertRC(rc);
4941
4942 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4943 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4944 Cannot do it here as we may have been long preempted. */
4945
4946#ifdef VBOX_STRICT
4947 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4948 pVmxTransient->uExitReason));
4949 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4950 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4951 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4952 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4953 else
4954 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4955 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4956 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4957
4958 /* VMX control bits. */
4959 uint32_t u32Val;
4960 uint64_t u64Val;
4961 HMVMXHCUINTREG uHCReg;
4962 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4963 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4964 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4965 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4966 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4967 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4968 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4969 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4970 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4971 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4972 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4973 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4974 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4975 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4976 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4977 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4978 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4979 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4980 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4981 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4982 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4983 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4984 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4985 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4994 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4995 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4996 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4997 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4998 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4999 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5000 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5001 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5002 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5003 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5004
5005 /* Guest bits. */
5006 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5007 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5008 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5009 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5011 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5012 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
5013 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
5014
5015 /* Host bits. */
5016 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5017 Log4(("Host CR0 %#RHr\n", uHCReg));
5018 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5019 Log4(("Host CR3 %#RHr\n", uHCReg));
5020 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5021 Log4(("Host CR4 %#RHr\n", uHCReg));
5022
5023 RTGDTR HostGdtr;
5024 PCX86DESCHC pDesc;
5025 ASMGetGDTR(&HostGdtr);
5026 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
5027 Log4(("Host CS %#08x\n", u32Val));
5028 if (u32Val < HostGdtr.cbGdt)
5029 {
5030 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5031 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
5032 }
5033
5034 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
5035 Log4(("Host DS %#08x\n", u32Val));
5036 if (u32Val < HostGdtr.cbGdt)
5037 {
5038 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5039 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
5040 }
5041
5042 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
5043 Log4(("Host ES %#08x\n", u32Val));
5044 if (u32Val < HostGdtr.cbGdt)
5045 {
5046 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5047 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
5048 }
5049
5050 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
5051 Log4(("Host FS %#08x\n", u32Val));
5052 if (u32Val < HostGdtr.cbGdt)
5053 {
5054 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5055 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
5056 }
5057
5058 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
5059 Log4(("Host GS %#08x\n", u32Val));
5060 if (u32Val < HostGdtr.cbGdt)
5061 {
5062 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5063 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
5064 }
5065
5066 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
5067 Log4(("Host SS %#08x\n", u32Val));
5068 if (u32Val < HostGdtr.cbGdt)
5069 {
5070 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5071 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
5072 }
5073
5074 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
5075 Log4(("Host TR %#08x\n", u32Val));
5076 if (u32Val < HostGdtr.cbGdt)
5077 {
5078 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5079 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5080 }
5081
5082 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5083 Log4(("Host TR Base %#RHv\n", uHCReg));
5084 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5085 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5086 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5087 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5088 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5089 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5091 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5093 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5095 Log4(("Host RSP %#RHv\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5097 Log4(("Host RIP %#RHv\n", uHCReg));
5098# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5099 if (HMVMX_IS_64BIT_HOST_MODE())
5100 {
5101 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5102 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5103 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5104 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5105 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5106 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5107 }
5108# endif
5109#endif /* VBOX_STRICT */
5110 break;
5111 }
5112
5113 default:
5114 /* Impossible */
5115 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5116 break;
5117 }
5118 NOREF(pVM); NOREF(pCtx);
5119}
5120
5121
5122#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5123#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5124# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5125#endif
5126#ifdef VBOX_STRICT
5127static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5128{
5129 switch (idxField)
5130 {
5131 case VMX_VMCS_GUEST_RIP:
5132 case VMX_VMCS_GUEST_RSP:
5133 case VMX_VMCS_GUEST_SYSENTER_EIP:
5134 case VMX_VMCS_GUEST_SYSENTER_ESP:
5135 case VMX_VMCS_GUEST_GDTR_BASE:
5136 case VMX_VMCS_GUEST_IDTR_BASE:
5137 case VMX_VMCS_GUEST_CS_BASE:
5138 case VMX_VMCS_GUEST_DS_BASE:
5139 case VMX_VMCS_GUEST_ES_BASE:
5140 case VMX_VMCS_GUEST_FS_BASE:
5141 case VMX_VMCS_GUEST_GS_BASE:
5142 case VMX_VMCS_GUEST_SS_BASE:
5143 case VMX_VMCS_GUEST_LDTR_BASE:
5144 case VMX_VMCS_GUEST_TR_BASE:
5145 case VMX_VMCS_GUEST_CR3:
5146 return true;
5147 }
5148 return false;
5149}
5150
5151static bool hmR0VmxIsValidReadField(uint32_t idxField)
5152{
5153 switch (idxField)
5154 {
5155 /* Read-only fields. */
5156 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5157 return true;
5158 }
5159 /* Remaining readable fields should also be writable. */
5160 return hmR0VmxIsValidWriteField(idxField);
5161}
5162#endif /* VBOX_STRICT */
5163
5164
5165/**
5166 * Executes the specified handler in 64-bit mode.
5167 *
5168 * @returns VBox status code.
5169 * @param pVM Pointer to the VM.
5170 * @param pVCpu Pointer to the VMCPU.
5171 * @param pCtx Pointer to the guest CPU context.
5172 * @param enmOp The operation to perform.
5173 * @param cbParam Number of parameters.
5174 * @param paParam Array of 32-bit parameters.
5175 */
5176VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5177 uint32_t *paParam)
5178{
5179 int rc, rc2;
5180 PHMGLOBALCPUINFO pCpu;
5181 RTHCPHYS HCPhysCpuPage;
5182 RTCCUINTREG uOldEflags;
5183
5184 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5185 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5186 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5187 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5188
5189#ifdef VBOX_STRICT
5190 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5191 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5192
5193 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5194 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5195#endif
5196
5197 /* Disable interrupts. */
5198 uOldEflags = ASMIntDisableFlags();
5199
5200#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5201 RTCPUID idHostCpu = RTMpCpuId();
5202 CPUMR0SetLApic(pVCpu, idHostCpu);
5203#endif
5204
5205 pCpu = HMR0GetCurrentCpu();
5206 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5207
5208 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5209 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5210
5211 /* Leave VMX Root Mode. */
5212 VMXDisable();
5213
5214 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5215
5216 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5217 CPUMSetHyperEIP(pVCpu, enmOp);
5218 for (int i = (int)cbParam - 1; i >= 0; i--)
5219 CPUMPushHyper(pVCpu, paParam[i]);
5220
5221 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5222
5223 /* Call the switcher. */
5224 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5225 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5226
5227 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5228 /* Make sure the VMX instructions don't cause #UD faults. */
5229 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5230
5231 /* Re-enter VMX Root Mode */
5232 rc2 = VMXEnable(HCPhysCpuPage);
5233 if (RT_FAILURE(rc2))
5234 {
5235 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5236 ASMSetFlags(uOldEflags);
5237 return rc2;
5238 }
5239
5240 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5241 AssertRC(rc2);
5242 Assert(!(ASMGetFlags() & X86_EFL_IF));
5243 ASMSetFlags(uOldEflags);
5244 return rc;
5245}
5246
5247
5248/**
5249 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5250 * supporting 64-bit guests.
5251 *
5252 * @returns VBox status code.
5253 * @param fResume Whether to VMLAUNCH or VMRESUME.
5254 * @param pCtx Pointer to the guest-CPU context.
5255 * @param pCache Pointer to the VMCS cache.
5256 * @param pVM Pointer to the VM.
5257 * @param pVCpu Pointer to the VMCPU.
5258 */
5259DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5260{
5261 uint32_t aParam[6];
5262 PHMGLOBALCPUINFO pCpu = NULL;
5263 RTHCPHYS HCPhysCpuPage = 0;
5264 int rc = VERR_INTERNAL_ERROR_5;
5265
5266 pCpu = HMR0GetCurrentCpu();
5267 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5268
5269#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5270 pCache->uPos = 1;
5271 pCache->interPD = PGMGetInterPaeCR3(pVM);
5272 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5273#endif
5274
5275#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5276 pCache->TestIn.HCPhysCpuPage = 0;
5277 pCache->TestIn.HCPhysVmcs = 0;
5278 pCache->TestIn.pCache = 0;
5279 pCache->TestOut.HCPhysVmcs = 0;
5280 pCache->TestOut.pCache = 0;
5281 pCache->TestOut.pCtx = 0;
5282 pCache->TestOut.eflags = 0;
5283#endif
5284
5285 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5286 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5287 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5288 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5289 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5290 aParam[5] = 0;
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5294 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5295#endif
5296 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5297
5298#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5299 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5300 Assert(pCtx->dr[4] == 10);
5301 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5302#endif
5303
5304#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5305 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5306 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5307 pVCpu->hm.s.vmx.HCPhysVmcs));
5308 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5309 pCache->TestOut.HCPhysVmcs));
5310 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5311 pCache->TestOut.pCache));
5312 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5313 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5314 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5315 pCache->TestOut.pCtx));
5316 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5317#endif
5318 return rc;
5319}
5320
5321
5322/**
5323 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5324 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5325 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5326 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5327 *
5328 * @returns VBox status code.
5329 * @param pVM Pointer to the VM.
5330 * @param pVCpu Pointer to the VMCPU.
5331 */
5332static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5333{
5334#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5335{ \
5336 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5337 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5338 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5339 ++cReadFields; \
5340}
5341
5342 AssertPtr(pVM);
5343 AssertPtr(pVCpu);
5344 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5345 uint32_t cReadFields = 0;
5346
5347 /*
5348 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5349 * and serve to indicate exceptions to the rules.
5350 */
5351
5352 /* Guest-natural selector base fields. */
5353#if 0
5354 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5355 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5356 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5357#endif
5358 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5359 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5360 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5361 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5362 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5363 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5364 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5365 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5366 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5367 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5368 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5369 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5370#if 0
5371 /* Unused natural width guest-state fields. */
5372 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5373 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5374#endif
5375 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5376 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5377
5378 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5379#if 0
5380 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5381 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5382 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5383 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5384 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5389#endif
5390
5391 /* Natural width guest-state fields. */
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5393#if 0
5394 /* Currently unused field. */
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5396#endif
5397
5398 if (pVM->hm.s.fNestedPaging)
5399 {
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5401 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5402 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5403 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5404 }
5405 else
5406 {
5407 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5408 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5409 }
5410
5411#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5412 return VINF_SUCCESS;
5413}
5414
5415
5416/**
5417 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5418 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5419 * darwin, running 64-bit guests).
5420 *
5421 * @returns VBox status code.
5422 * @param pVCpu Pointer to the VMCPU.
5423 * @param idxField The VMCS field encoding.
5424 * @param u64Val 16, 32 or 64-bit value.
5425 */
5426VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5427{
5428 int rc;
5429 switch (idxField)
5430 {
5431 /*
5432 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5433 */
5434 /* 64-bit Control fields. */
5435 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5436 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5437 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5438 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5439 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5440 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5441 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5442 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5443 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5444 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5445 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5446 case VMX_VMCS64_CTRL_EPTP_FULL:
5447 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5448 /* 64-bit Guest-state fields. */
5449 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5450 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5451 case VMX_VMCS64_GUEST_PAT_FULL:
5452 case VMX_VMCS64_GUEST_EFER_FULL:
5453 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5454 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5455 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5456 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5457 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5458 /* 64-bit Host-state fields. */
5459 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5460 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5461 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5462 {
5463 rc = VMXWriteVmcs32(idxField, u64Val);
5464 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5465 break;
5466 }
5467
5468 /*
5469 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5470 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5471 */
5472 /* Natural-width Guest-state fields. */
5473 case VMX_VMCS_GUEST_CR3:
5474 case VMX_VMCS_GUEST_ES_BASE:
5475 case VMX_VMCS_GUEST_CS_BASE:
5476 case VMX_VMCS_GUEST_SS_BASE:
5477 case VMX_VMCS_GUEST_DS_BASE:
5478 case VMX_VMCS_GUEST_FS_BASE:
5479 case VMX_VMCS_GUEST_GS_BASE:
5480 case VMX_VMCS_GUEST_LDTR_BASE:
5481 case VMX_VMCS_GUEST_TR_BASE:
5482 case VMX_VMCS_GUEST_GDTR_BASE:
5483 case VMX_VMCS_GUEST_IDTR_BASE:
5484 case VMX_VMCS_GUEST_RSP:
5485 case VMX_VMCS_GUEST_RIP:
5486 case VMX_VMCS_GUEST_SYSENTER_ESP:
5487 case VMX_VMCS_GUEST_SYSENTER_EIP:
5488 {
5489 if (!(u64Val >> 32))
5490 {
5491 /* If this field is 64-bit, VT-x will zero out the top bits. */
5492 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5493 }
5494 else
5495 {
5496 /* Assert that only the 32->64 switcher case should ever come here. */
5497 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5498 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5499 }
5500 break;
5501 }
5502
5503 default:
5504 {
5505 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5506 rc = VERR_INVALID_PARAMETER;
5507 break;
5508 }
5509 }
5510 AssertRCReturn(rc, rc);
5511 return rc;
5512}
5513
5514
5515/**
5516 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5517 * hosts (except darwin) for 64-bit guests.
5518 *
5519 * @param pVCpu Pointer to the VMCPU.
5520 * @param idxField The VMCS field encoding.
5521 * @param u64Val 16, 32 or 64-bit value.
5522 */
5523VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5524{
5525 AssertPtr(pVCpu);
5526 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5527
5528 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5529 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5530
5531 /* Make sure there are no duplicates. */
5532 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5533 {
5534 if (pCache->Write.aField[i] == idxField)
5535 {
5536 pCache->Write.aFieldVal[i] = u64Val;
5537 return VINF_SUCCESS;
5538 }
5539 }
5540
5541 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5542 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5543 pCache->Write.cValidEntries++;
5544 return VINF_SUCCESS;
5545}
5546
5547/* Enable later when the assembly code uses these as callbacks. */
5548#if 0
5549/*
5550 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5551 *
5552 * @param pVCpu Pointer to the VMCPU.
5553 * @param pCache Pointer to the VMCS cache.
5554 *
5555 * @remarks No-long-jump zone!!!
5556 */
5557VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5558{
5559 AssertPtr(pCache);
5560 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5561 {
5562 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5563 AssertRC(rc);
5564 }
5565 pCache->Write.cValidEntries = 0;
5566}
5567
5568
5569/**
5570 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5571 *
5572 * @param pVCpu Pointer to the VMCPU.
5573 * @param pCache Pointer to the VMCS cache.
5574 *
5575 * @remarks No-long-jump zone!!!
5576 */
5577VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5578{
5579 AssertPtr(pCache);
5580 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5581 {
5582 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5583 AssertRC(rc);
5584 }
5585}
5586#endif
5587#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5588
5589
5590/**
5591 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5592 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5593 * timer.
5594 *
5595 * @returns VBox status code.
5596 * @param pVCpu Pointer to the VMCPU.
5597 *
5598 * @remarks No-long-jump zone!!!
5599 */
5600static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5601{
5602 int rc = VERR_INTERNAL_ERROR_5;
5603 bool fOffsettedTsc = false;
5604 bool fParavirtTsc = false;
5605 PVM pVM = pVCpu->CTX_SUFF(pVM);
5606 if (pVM->hm.s.vmx.fUsePreemptTimer)
5607 {
5608 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &fParavirtTsc,
5609 &pVCpu->hm.s.vmx.u64TSCOffset);
5610
5611 /* Make sure the returned values have sane upper and lower boundaries. */
5612 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5613 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5614 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5615 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5616
5617 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5619 }
5620 else
5621 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5622
5623#if 1
5624 if (fParavirtTsc)
5625 {
5626#if 1
5627 uint64_t const u64CurTsc = ASMReadTSC();
5628 uint64_t const u64LastTick = TMCpuTickGetLastSeen(pVCpu);
5629 if (u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset < u64LastTick)
5630 {
5631 pVCpu->hm.s.vmx.u64TSCOffset = (u64LastTick - u64CurTsc);
5632 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffsetAdjusted);
5633 }
5634
5635 Assert(u64CurTsc + pVCpu->hm.s.vmx.u64TSCOffset >= u64LastTick);
5636#endif
5637 rc = GIMR0UpdateParavirtTsc(pVM, pVCpu->hm.s.vmx.u64TSCOffset);
5638 AssertRC(rc);
5639 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5640 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRC(rc);
5641
5642 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5643 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5644 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5645 }
5646 else
5647#else
5648 if (fParavirtTsc)
5649 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5650#endif
5651 if (fOffsettedTsc)
5652 {
5653 uint64_t u64CurTSC = ASMReadTSC();
5654 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5655 {
5656 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5657 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5658
5659 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5660 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5661 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5662 }
5663 else
5664 {
5665 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5666 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5667 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5668 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5669 }
5670 }
5671 else
5672 {
5673 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5674 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5675 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5676 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5677 }
5678}
5679
5680
5681/**
5682 * Determines if an exception is a contributory exception. Contributory
5683 * exceptions are ones which can cause double-faults. Page-fault is
5684 * intentionally not included here as it's a conditional contributory exception.
5685 *
5686 * @returns true if the exception is contributory, false otherwise.
5687 * @param uVector The exception vector.
5688 */
5689DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5690{
5691 switch (uVector)
5692 {
5693 case X86_XCPT_GP:
5694 case X86_XCPT_SS:
5695 case X86_XCPT_NP:
5696 case X86_XCPT_TS:
5697 case X86_XCPT_DE:
5698 return true;
5699 default:
5700 break;
5701 }
5702 return false;
5703}
5704
5705
5706/**
5707 * Sets an event as a pending event to be injected into the guest.
5708 *
5709 * @param pVCpu Pointer to the VMCPU.
5710 * @param u32IntInfo The VM-entry interruption-information field.
5711 * @param cbInstr The VM-entry instruction length in bytes (for software
5712 * interrupts, exceptions and privileged software
5713 * exceptions).
5714 * @param u32ErrCode The VM-entry exception error code.
5715 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5716 * page-fault.
5717 *
5718 * @remarks Statistics counter assumes this is a guest event being injected or
5719 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5720 * always incremented.
5721 */
5722DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5723 RTGCUINTPTR GCPtrFaultAddress)
5724{
5725 Assert(!pVCpu->hm.s.Event.fPending);
5726 pVCpu->hm.s.Event.fPending = true;
5727 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5728 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5729 pVCpu->hm.s.Event.cbInstr = cbInstr;
5730 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5731
5732 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5733}
5734
5735
5736/**
5737 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5738 *
5739 * @param pVCpu Pointer to the VMCPU.
5740 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5741 * out-of-sync. Make sure to update the required fields
5742 * before using them.
5743 */
5744DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5745{
5746 NOREF(pMixedCtx);
5747 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5748 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5749 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5750 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5751}
5752
5753
5754/**
5755 * Handle a condition that occurred while delivering an event through the guest
5756 * IDT.
5757 *
5758 * @returns VBox status code (informational error codes included).
5759 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5760 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5761 * continue execution of the guest which will delivery the #DF.
5762 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5763 *
5764 * @param pVCpu Pointer to the VMCPU.
5765 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5766 * out-of-sync. Make sure to update the required fields
5767 * before using them.
5768 * @param pVmxTransient Pointer to the VMX transient structure.
5769 *
5770 * @remarks No-long-jump zone!!!
5771 */
5772static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5773{
5774 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5775
5776 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5777 AssertRCReturn(rc, rc);
5778 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5779 {
5780 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5781 AssertRCReturn(rc, rc);
5782
5783 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5784 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5785
5786 typedef enum
5787 {
5788 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5789 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5790 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5791 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5792 } VMXREFLECTXCPT;
5793
5794 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5795 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5796 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5797 {
5798 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5799 {
5800 enmReflect = VMXREFLECTXCPT_XCPT;
5801#ifdef VBOX_STRICT
5802 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5803 && uExitVector == X86_XCPT_PF)
5804 {
5805 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5806 }
5807#endif
5808 if ( uExitVector == X86_XCPT_PF
5809 && uIdtVector == X86_XCPT_PF)
5810 {
5811 pVmxTransient->fVectoringPF = true;
5812 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5813 }
5814 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5815 && hmR0VmxIsContributoryXcpt(uExitVector)
5816 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5817 || uIdtVector == X86_XCPT_PF))
5818 {
5819 enmReflect = VMXREFLECTXCPT_DF;
5820 }
5821 else if (uIdtVector == X86_XCPT_DF)
5822 enmReflect = VMXREFLECTXCPT_TF;
5823 }
5824 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5825 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5826 {
5827 /*
5828 * If this exception occurred while delivering the NMI, we need to clear the block-by-NMI field in the
5829 * guest interruptibility-state before re-delivering the NMI, otherwise the subsequent VM-entry would fail.
5830 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5831 */
5832 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
5833 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5834 enmReflect = VMXREFLECTXCPT_XCPT;
5835 }
5836 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5837 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5838 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5839 {
5840 /*
5841 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5842 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5843 */
5844 enmReflect = VMXREFLECTXCPT_XCPT;
5845 }
5846 }
5847 else
5848 {
5849 /*
5850 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5851 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5852 * original exception to the guest after handling the VM-exit.
5853 */
5854 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5855 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5856 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5857 {
5858 enmReflect = VMXREFLECTXCPT_XCPT;
5859 }
5860 }
5861
5862 switch (enmReflect)
5863 {
5864 case VMXREFLECTXCPT_XCPT:
5865 {
5866 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5867 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5868 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5869
5870 uint32_t u32ErrCode = 0;
5871 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5872 {
5873 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5874 AssertRCReturn(rc, rc);
5875 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5876 }
5877
5878 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5879 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5880 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5881 rc = VINF_SUCCESS;
5882 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5883 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5884
5885 break;
5886 }
5887
5888 case VMXREFLECTXCPT_DF:
5889 {
5890 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5891 rc = VINF_HM_DOUBLE_FAULT;
5892 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5893 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5894
5895 break;
5896 }
5897
5898 case VMXREFLECTXCPT_TF:
5899 {
5900 rc = VINF_EM_RESET;
5901 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5902 uExitVector));
5903 break;
5904 }
5905
5906 default:
5907 Assert(rc == VINF_SUCCESS);
5908 break;
5909 }
5910 }
5911 else if ( VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5912 && uExitVector != X86_XCPT_DF
5913 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5914 {
5915 /*
5916 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5917 * We need to reset the block-by-NMI field so that NMIs remain blocked until the IRET execution is completed.
5918 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5919 */
5920 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5921 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5922 }
5923
5924 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5925 return rc;
5926}
5927
5928
5929/**
5930 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5931 *
5932 * @returns VBox status code.
5933 * @param pVCpu Pointer to the VMCPU.
5934 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5935 * out-of-sync. Make sure to update the required fields
5936 * before using them.
5937 *
5938 * @remarks No-long-jump zone!!!
5939 */
5940static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5941{
5942 NOREF(pMixedCtx);
5943
5944 /*
5945 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5946 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5947 */
5948 VMMRZCallRing3Disable(pVCpu);
5949 HM_DISABLE_PREEMPT_IF_NEEDED();
5950
5951 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5952 {
5953 uint32_t uVal = 0;
5954 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5955 AssertRCReturn(rc, rc);
5956
5957 uint32_t uShadow = 0;
5958 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5959 AssertRCReturn(rc, rc);
5960
5961 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5962 CPUMSetGuestCR0(pVCpu, uVal);
5963 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5964 }
5965
5966 HM_RESTORE_PREEMPT_IF_NEEDED();
5967 VMMRZCallRing3Enable(pVCpu);
5968 return VINF_SUCCESS;
5969}
5970
5971
5972/**
5973 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5974 *
5975 * @returns VBox status code.
5976 * @param pVCpu Pointer to the VMCPU.
5977 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5978 * out-of-sync. Make sure to update the required fields
5979 * before using them.
5980 *
5981 * @remarks No-long-jump zone!!!
5982 */
5983static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5984{
5985 NOREF(pMixedCtx);
5986
5987 int rc = VINF_SUCCESS;
5988 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5989 {
5990 uint32_t uVal = 0;
5991 uint32_t uShadow = 0;
5992 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5993 AssertRCReturn(rc, rc);
5994 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5995 AssertRCReturn(rc, rc);
5996
5997 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5998 CPUMSetGuestCR4(pVCpu, uVal);
5999 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6000 }
6001 return rc;
6002}
6003
6004
6005/**
6006 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6007 *
6008 * @returns VBox status code.
6009 * @param pVCpu Pointer to the VMCPU.
6010 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6011 * out-of-sync. Make sure to update the required fields
6012 * before using them.
6013 *
6014 * @remarks No-long-jump zone!!!
6015 */
6016static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6017{
6018 int rc = VINF_SUCCESS;
6019 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6020 {
6021 uint64_t u64Val = 0;
6022 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6023 AssertRCReturn(rc, rc);
6024
6025 pMixedCtx->rip = u64Val;
6026 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6027 }
6028 return rc;
6029}
6030
6031
6032/**
6033 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6034 *
6035 * @returns VBox status code.
6036 * @param pVCpu Pointer to the VMCPU.
6037 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6038 * out-of-sync. Make sure to update the required fields
6039 * before using them.
6040 *
6041 * @remarks No-long-jump zone!!!
6042 */
6043static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6044{
6045 int rc = VINF_SUCCESS;
6046 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6047 {
6048 uint64_t u64Val = 0;
6049 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6050 AssertRCReturn(rc, rc);
6051
6052 pMixedCtx->rsp = u64Val;
6053 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6054 }
6055 return rc;
6056}
6057
6058
6059/**
6060 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6061 *
6062 * @returns VBox status code.
6063 * @param pVCpu Pointer to the VMCPU.
6064 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6065 * out-of-sync. Make sure to update the required fields
6066 * before using them.
6067 *
6068 * @remarks No-long-jump zone!!!
6069 */
6070static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6071{
6072 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6073 {
6074 uint32_t uVal = 0;
6075 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6076 AssertRCReturn(rc, rc);
6077
6078 pMixedCtx->eflags.u32 = uVal;
6079 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6080 {
6081 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6082 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6083
6084 pMixedCtx->eflags.Bits.u1VM = 0;
6085 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6086 }
6087
6088 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6089 }
6090 return VINF_SUCCESS;
6091}
6092
6093
6094/**
6095 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6096 * guest-CPU context.
6097 */
6098DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6099{
6100 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6101 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6102 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6103 return rc;
6104}
6105
6106
6107/**
6108 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6109 * from the guest-state area in the VMCS.
6110 *
6111 * @param pVCpu Pointer to the VMCPU.
6112 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6113 * out-of-sync. Make sure to update the required fields
6114 * before using them.
6115 *
6116 * @remarks No-long-jump zone!!!
6117 */
6118static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6119{
6120 uint32_t uIntrState = 0;
6121 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6122 AssertRC(rc);
6123
6124 if (!uIntrState)
6125 {
6126 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6127 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6128
6129 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6130 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6131 }
6132 else
6133 {
6134 if ( (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
6135 || (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6136 {
6137 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6138 AssertRC(rc);
6139 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6140 AssertRC(rc);
6141
6142 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6143 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6144 }
6145
6146 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6147 {
6148 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6149 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6150 }
6151 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6152 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6153 }
6154}
6155
6156
6157/**
6158 * Saves the guest's activity state.
6159 *
6160 * @returns VBox status code.
6161 * @param pVCpu Pointer to the VMCPU.
6162 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6163 * out-of-sync. Make sure to update the required fields
6164 * before using them.
6165 *
6166 * @remarks No-long-jump zone!!!
6167 */
6168static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6169{
6170 NOREF(pMixedCtx);
6171 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6172 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6173 return VINF_SUCCESS;
6174}
6175
6176
6177/**
6178 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6179 * the current VMCS into the guest-CPU context.
6180 *
6181 * @returns VBox status code.
6182 * @param pVCpu Pointer to the VMCPU.
6183 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6184 * out-of-sync. Make sure to update the required fields
6185 * before using them.
6186 *
6187 * @remarks No-long-jump zone!!!
6188 */
6189static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6190{
6191 int rc = VINF_SUCCESS;
6192 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6193 {
6194 uint32_t u32Val = 0;
6195 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6196 pMixedCtx->SysEnter.cs = u32Val;
6197 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6198 }
6199
6200 uint64_t u64Val = 0;
6201 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6202 {
6203 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6204 pMixedCtx->SysEnter.eip = u64Val;
6205 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6206 }
6207 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6208 {
6209 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6210 pMixedCtx->SysEnter.esp = u64Val;
6211 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6212 }
6213 return rc;
6214}
6215
6216
6217/**
6218 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6219 * the CPU back into the guest-CPU context.
6220 *
6221 * @returns VBox status code.
6222 * @param pVCpu Pointer to the VMCPU.
6223 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6224 * out-of-sync. Make sure to update the required fields
6225 * before using them.
6226 *
6227 * @remarks No-long-jump zone!!!
6228 */
6229static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6230{
6231#if HC_ARCH_BITS == 64
6232 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6233 {
6234 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6235 VMMRZCallRing3Disable(pVCpu);
6236 HM_DISABLE_PREEMPT_IF_NEEDED();
6237
6238 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6239 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6240 {
6241 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6242 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6243 }
6244
6245 HM_RESTORE_PREEMPT_IF_NEEDED();
6246 VMMRZCallRing3Enable(pVCpu);
6247 }
6248 else
6249 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6250#else
6251 NOREF(pMixedCtx);
6252 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6253#endif
6254
6255 return VINF_SUCCESS;
6256}
6257
6258
6259/**
6260 * Saves the auto load/store'd guest MSRs from the current VMCS into
6261 * the guest-CPU context.
6262 *
6263 * @returns VBox status code.
6264 * @param pVCpu Pointer to the VMCPU.
6265 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6266 * out-of-sync. Make sure to update the required fields
6267 * before using them.
6268 *
6269 * @remarks No-long-jump zone!!!
6270 */
6271static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6272{
6273 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6274 return VINF_SUCCESS;
6275
6276 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6277 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6278 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6279 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6280 {
6281 switch (pMsr->u32Msr)
6282 {
6283 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6284 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6285 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6286 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6287 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6288 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6289 break;
6290
6291 default:
6292 {
6293 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6294 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6295 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6296 }
6297 }
6298 }
6299
6300 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6301 return VINF_SUCCESS;
6302}
6303
6304
6305/**
6306 * Saves the guest control registers from the current VMCS into the guest-CPU
6307 * context.
6308 *
6309 * @returns VBox status code.
6310 * @param pVCpu Pointer to the VMCPU.
6311 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6312 * out-of-sync. Make sure to update the required fields
6313 * before using them.
6314 *
6315 * @remarks No-long-jump zone!!!
6316 */
6317static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6318{
6319 /* Guest CR0. Guest FPU. */
6320 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6321 AssertRCReturn(rc, rc);
6322
6323 /* Guest CR4. */
6324 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6325 AssertRCReturn(rc, rc);
6326
6327 /* Guest CR2 - updated always during the world-switch or in #PF. */
6328 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6329 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6330 {
6331 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6332 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6333
6334 PVM pVM = pVCpu->CTX_SUFF(pVM);
6335 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6336 || ( pVM->hm.s.fNestedPaging
6337 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6338 {
6339 uint64_t u64Val = 0;
6340 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6341 if (pMixedCtx->cr3 != u64Val)
6342 {
6343 CPUMSetGuestCR3(pVCpu, u64Val);
6344 if (VMMRZCallRing3IsEnabled(pVCpu))
6345 {
6346 PGMUpdateCR3(pVCpu, u64Val);
6347 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6348 }
6349 else
6350 {
6351 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6352 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6353 }
6354 }
6355
6356 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6357 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6358 {
6359 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6360 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6361 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6362 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6363
6364 if (VMMRZCallRing3IsEnabled(pVCpu))
6365 {
6366 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6367 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6368 }
6369 else
6370 {
6371 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6372 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6373 }
6374 }
6375 }
6376
6377 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6378 }
6379
6380 /*
6381 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6382 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6383 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6384 *
6385 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6386 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6387 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6388 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6389 *
6390 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6391 */
6392 if (VMMRZCallRing3IsEnabled(pVCpu))
6393 {
6394 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6395 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6396
6397 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6398 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6399
6400 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6401 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6402 }
6403
6404 return rc;
6405}
6406
6407
6408/**
6409 * Reads a guest segment register from the current VMCS into the guest-CPU
6410 * context.
6411 *
6412 * @returns VBox status code.
6413 * @param pVCpu Pointer to the VMCPU.
6414 * @param idxSel Index of the selector in the VMCS.
6415 * @param idxLimit Index of the segment limit in the VMCS.
6416 * @param idxBase Index of the segment base in the VMCS.
6417 * @param idxAccess Index of the access rights of the segment in the VMCS.
6418 * @param pSelReg Pointer to the segment selector.
6419 *
6420 * @remarks No-long-jump zone!!!
6421 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6422 * macro as that takes care of whether to read from the VMCS cache or
6423 * not.
6424 */
6425DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6426 PCPUMSELREG pSelReg)
6427{
6428 NOREF(pVCpu);
6429
6430 uint32_t u32Val = 0;
6431 int rc = VMXReadVmcs32(idxSel, &u32Val);
6432 AssertRCReturn(rc, rc);
6433 pSelReg->Sel = (uint16_t)u32Val;
6434 pSelReg->ValidSel = (uint16_t)u32Val;
6435 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6436
6437 rc = VMXReadVmcs32(idxLimit, &u32Val);
6438 AssertRCReturn(rc, rc);
6439 pSelReg->u32Limit = u32Val;
6440
6441 uint64_t u64Val = 0;
6442 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6443 AssertRCReturn(rc, rc);
6444 pSelReg->u64Base = u64Val;
6445
6446 rc = VMXReadVmcs32(idxAccess, &u32Val);
6447 AssertRCReturn(rc, rc);
6448 pSelReg->Attr.u = u32Val;
6449
6450 /*
6451 * If VT-x marks the segment as unusable, most other bits remain undefined:
6452 * - For CS the L, D and G bits have meaning.
6453 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6454 * - For the remaining data segments no bits are defined.
6455 *
6456 * The present bit and the unusable bit has been observed to be set at the
6457 * same time (the selector was supposed to be invalid as we started executing
6458 * a V8086 interrupt in ring-0).
6459 *
6460 * What should be important for the rest of the VBox code, is that the P bit is
6461 * cleared. Some of the other VBox code recognizes the unusable bit, but
6462 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6463 * safe side here, we'll strip off P and other bits we don't care about. If
6464 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6465 *
6466 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6467 */
6468 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6469 {
6470 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6471
6472 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6473 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6474 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6475
6476 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6477#ifdef DEBUG_bird
6478 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6479 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6480 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6481#endif
6482 }
6483 return VINF_SUCCESS;
6484}
6485
6486
6487#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6488# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6489 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6490 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6491#else
6492# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6493 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6494 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6495#endif
6496
6497
6498/**
6499 * Saves the guest segment registers from the current VMCS into the guest-CPU
6500 * context.
6501 *
6502 * @returns VBox status code.
6503 * @param pVCpu Pointer to the VMCPU.
6504 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6505 * out-of-sync. Make sure to update the required fields
6506 * before using them.
6507 *
6508 * @remarks No-long-jump zone!!!
6509 */
6510static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6511{
6512 /* Guest segment registers. */
6513 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6514 {
6515 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6516 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6517 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6518 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6519 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6520 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6521 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6522
6523 /* Restore segment attributes for real-on-v86 mode hack. */
6524 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6525 {
6526 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6527 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6528 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6529 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6530 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6531 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6532 }
6533 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6534 }
6535
6536 return VINF_SUCCESS;
6537}
6538
6539
6540/**
6541 * Saves the guest descriptor table registers and task register from the current
6542 * VMCS into the guest-CPU context.
6543 *
6544 * @returns VBox status code.
6545 * @param pVCpu Pointer to the VMCPU.
6546 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6547 * out-of-sync. Make sure to update the required fields
6548 * before using them.
6549 *
6550 * @remarks No-long-jump zone!!!
6551 */
6552static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6553{
6554 int rc = VINF_SUCCESS;
6555
6556 /* Guest LDTR. */
6557 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6558 {
6559 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6560 AssertRCReturn(rc, rc);
6561 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6562 }
6563
6564 /* Guest GDTR. */
6565 uint64_t u64Val = 0;
6566 uint32_t u32Val = 0;
6567 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6568 {
6569 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6570 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6571 pMixedCtx->gdtr.pGdt = u64Val;
6572 pMixedCtx->gdtr.cbGdt = u32Val;
6573 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6574 }
6575
6576 /* Guest IDTR. */
6577 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6578 {
6579 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6580 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6581 pMixedCtx->idtr.pIdt = u64Val;
6582 pMixedCtx->idtr.cbIdt = u32Val;
6583 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6584 }
6585
6586 /* Guest TR. */
6587 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6588 {
6589 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6590 AssertRCReturn(rc, rc);
6591
6592 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6593 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6594 {
6595 rc = VMXLOCAL_READ_SEG(TR, tr);
6596 AssertRCReturn(rc, rc);
6597 }
6598 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6599 }
6600 return rc;
6601}
6602
6603#undef VMXLOCAL_READ_SEG
6604
6605
6606/**
6607 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6608 * context.
6609 *
6610 * @returns VBox status code.
6611 * @param pVCpu Pointer to the VMCPU.
6612 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6613 * out-of-sync. Make sure to update the required fields
6614 * before using them.
6615 *
6616 * @remarks No-long-jump zone!!!
6617 */
6618static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6619{
6620 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6621 {
6622 if (!pVCpu->hm.s.fUsingHyperDR7)
6623 {
6624 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6625 uint32_t u32Val;
6626 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6627 pMixedCtx->dr[7] = u32Val;
6628 }
6629
6630 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6631 }
6632 return VINF_SUCCESS;
6633}
6634
6635
6636/**
6637 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6638 *
6639 * @returns VBox status code.
6640 * @param pVCpu Pointer to the VMCPU.
6641 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6642 * out-of-sync. Make sure to update the required fields
6643 * before using them.
6644 *
6645 * @remarks No-long-jump zone!!!
6646 */
6647static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6648{
6649 NOREF(pMixedCtx);
6650
6651 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6652 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6653 return VINF_SUCCESS;
6654}
6655
6656
6657/**
6658 * Saves the entire guest state from the currently active VMCS into the
6659 * guest-CPU context. This essentially VMREADs all guest-data.
6660 *
6661 * @returns VBox status code.
6662 * @param pVCpu Pointer to the VMCPU.
6663 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6664 * out-of-sync. Make sure to update the required fields
6665 * before using them.
6666 */
6667static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6668{
6669 Assert(pVCpu);
6670 Assert(pMixedCtx);
6671
6672 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6673 return VINF_SUCCESS;
6674
6675 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6676 again on the ring-3 callback path, there is no real need to. */
6677 if (VMMRZCallRing3IsEnabled(pVCpu))
6678 VMMR0LogFlushDisable(pVCpu);
6679 else
6680 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6681 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6682
6683 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6684 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6685
6686 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6688
6689 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6690 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6691
6692 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6693 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6694
6695 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6696 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6697
6698 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6699 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6700
6701 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6702 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6703
6704 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6705 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6706
6707 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6708 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6709
6710 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6711 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6712
6713 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6714 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6715
6716 if (VMMRZCallRing3IsEnabled(pVCpu))
6717 VMMR0LogFlushEnable(pVCpu);
6718
6719 return rc;
6720}
6721
6722
6723/**
6724 * Check per-VM and per-VCPU force flag actions that require us to go back to
6725 * ring-3 for one reason or another.
6726 *
6727 * @returns VBox status code (information status code included).
6728 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6729 * ring-3.
6730 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6731 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6732 * interrupts)
6733 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6734 * all EMTs to be in ring-3.
6735 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6736 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6737 * to the EM loop.
6738 *
6739 * @param pVM Pointer to the VM.
6740 * @param pVCpu Pointer to the VMCPU.
6741 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6742 * out-of-sync. Make sure to update the required fields
6743 * before using them.
6744 */
6745static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6746{
6747 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6748
6749 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6750 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6751 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6752 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6753 {
6754 /* We need the control registers now, make sure the guest-CPU context is updated. */
6755 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6756 AssertRCReturn(rc3, rc3);
6757
6758 /* Pending HM CR3 sync. */
6759 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6760 {
6761 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6762 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6763 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6764 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6765 }
6766
6767 /* Pending HM PAE PDPEs. */
6768 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6769 {
6770 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6771 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6772 }
6773
6774 /* Pending PGM C3 sync. */
6775 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6776 {
6777 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6778 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6779 if (rc2 != VINF_SUCCESS)
6780 {
6781 AssertRC(rc2);
6782 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6783 return rc2;
6784 }
6785 }
6786
6787 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6788 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6789 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6790 {
6791 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6792 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6793 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6794 return rc2;
6795 }
6796
6797 /* Pending VM request packets, such as hardware interrupts. */
6798 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6799 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6800 {
6801 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6802 return VINF_EM_PENDING_REQUEST;
6803 }
6804
6805 /* Pending PGM pool flushes. */
6806 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6807 {
6808 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6809 return VINF_PGM_POOL_FLUSH_PENDING;
6810 }
6811
6812 /* Pending DMA requests. */
6813 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6814 {
6815 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6816 return VINF_EM_RAW_TO_R3;
6817 }
6818 }
6819
6820 return VINF_SUCCESS;
6821}
6822
6823
6824/**
6825 * Converts any TRPM trap into a pending HM event. This is typically used when
6826 * entering from ring-3 (not longjmp returns).
6827 *
6828 * @param pVCpu Pointer to the VMCPU.
6829 */
6830static int hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6831{
6832 Assert(TRPMHasTrap(pVCpu));
6833 Assert(!pVCpu->hm.s.Event.fPending);
6834
6835 uint8_t uVector;
6836 TRPMEVENT enmTrpmEvent;
6837 RTGCUINT uErrCode;
6838 RTGCUINTPTR GCPtrFaultAddress;
6839 uint8_t cbInstr;
6840
6841 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6842 AssertRC(rc);
6843
6844 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6845 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6846 if (enmTrpmEvent == TRPM_TRAP)
6847 {
6848 switch (uVector)
6849 {
6850 case X86_XCPT_BP:
6851 case X86_XCPT_OF:
6852 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6853 break;
6854
6855 case X86_XCPT_PF:
6856 case X86_XCPT_DF:
6857 case X86_XCPT_TS:
6858 case X86_XCPT_NP:
6859 case X86_XCPT_SS:
6860 case X86_XCPT_GP:
6861 case X86_XCPT_AC:
6862 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6863 /* no break! */
6864 default:
6865 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6866 break;
6867 }
6868 }
6869 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6870 {
6871 if (uVector == X86_XCPT_NMI)
6872 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6873 else
6874 {
6875 uint32_t uEFlags = CPUMGetGuestEFlags(pVCpu);
6876 if (!(uEFlags & X86_EFL_IF))
6877 return VERR_VMX_IPE_5;
6878 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6879 }
6880 }
6881 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6882 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6883 else
6884 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6885
6886 rc = TRPMResetTrap(pVCpu);
6887 AssertRC(rc);
6888 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6889 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6890
6891 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6892 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6893 return VINF_SUCCESS;
6894}
6895
6896
6897/**
6898 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6899 * VT-x to execute any instruction.
6900 *
6901 * @param pvCpu Pointer to the VMCPU.
6902 */
6903static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6904{
6905 Assert(pVCpu->hm.s.Event.fPending);
6906
6907 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6908 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6909 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6910 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6911
6912 /* If a trap was already pending, we did something wrong! */
6913 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6914
6915 TRPMEVENT enmTrapType;
6916 switch (uVectorType)
6917 {
6918 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6919 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6920 enmTrapType = TRPM_HARDWARE_INT;
6921 break;
6922
6923 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6924 enmTrapType = TRPM_SOFTWARE_INT;
6925 break;
6926
6927 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6928 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6929 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6930 enmTrapType = TRPM_TRAP;
6931 break;
6932
6933 default:
6934 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6935 enmTrapType = TRPM_32BIT_HACK;
6936 break;
6937 }
6938
6939 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6940
6941 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6942 AssertRC(rc);
6943
6944 if (fErrorCodeValid)
6945 TRPMSetErrorCode(pVCpu, uErrorCode);
6946
6947 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6948 && uVector == X86_XCPT_PF)
6949 {
6950 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6951 }
6952 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6953 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6954 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6955 {
6956 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6957 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6958 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6959 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6960 }
6961 pVCpu->hm.s.Event.fPending = false;
6962}
6963
6964
6965/**
6966 * Does the necessary state syncing before returning to ring-3 for any reason
6967 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6968 *
6969 * @returns VBox status code.
6970 * @param pVM Pointer to the VM.
6971 * @param pVCpu Pointer to the VMCPU.
6972 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6973 * be out-of-sync. Make sure to update the required
6974 * fields before using them.
6975 * @param fSaveGuestState Whether to save the guest state or not.
6976 *
6977 * @remarks No-long-jmp zone!!!
6978 */
6979static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6980{
6981 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6982 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6983
6984 RTCPUID idCpu = RTMpCpuId();
6985 Log4Func(("HostCpuId=%u\n", idCpu));
6986
6987 /*
6988 * !!! IMPORTANT !!!
6989 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6990 */
6991
6992 /* Save the guest state if necessary. */
6993 if ( fSaveGuestState
6994 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6995 {
6996 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6997 AssertRCReturn(rc, rc);
6998 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6999 }
7000
7001 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7002 if (CPUMIsGuestFPUStateActive(pVCpu))
7003 {
7004 /* We shouldn't reload CR0 without saving it first. */
7005 if (!fSaveGuestState)
7006 {
7007 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7008 AssertRCReturn(rc, rc);
7009 }
7010 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
7011 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
7012 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7013 }
7014
7015 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7016#ifdef VBOX_STRICT
7017 if (CPUMIsHyperDebugStateActive(pVCpu))
7018 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7019#endif
7020 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7021 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7022 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7023 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7024
7025#if HC_ARCH_BITS == 64
7026 /* Restore host-state bits that VT-x only restores partially. */
7027 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7028 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7029 {
7030 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7031 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7032 }
7033 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7034#endif
7035
7036#if HC_ARCH_BITS == 64
7037 /* Restore the host MSRs as we're leaving VT-x context. */
7038 if ( pVM->hm.s.fAllow64BitGuests
7039 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7040 {
7041 /* We shouldn't reload the guest MSRs without saving it first. */
7042 if (!fSaveGuestState)
7043 {
7044 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7045 AssertRCReturn(rc, rc);
7046 }
7047 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7048 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7049 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
7050 }
7051#endif
7052
7053 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7054 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7055
7056 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7057 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7058 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7059 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7060 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7061 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7062 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7063 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7064
7065 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7066
7067 /** @todo This partially defeats the purpose of having preemption hooks.
7068 * The problem is, deregistering the hooks should be moved to a place that
7069 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7070 * context.
7071 */
7072 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7073 {
7074 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7075 AssertRCReturn(rc, rc);
7076
7077 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7078 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7079 }
7080 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7081 NOREF(idCpu);
7082
7083 return VINF_SUCCESS;
7084}
7085
7086
7087/**
7088 * Leaves the VT-x session.
7089 *
7090 * @returns VBox status code.
7091 * @param pVM Pointer to the VM.
7092 * @param pVCpu Pointer to the VMCPU.
7093 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7094 * out-of-sync. Make sure to update the required fields
7095 * before using them.
7096 *
7097 * @remarks No-long-jmp zone!!!
7098 */
7099DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7100{
7101 HM_DISABLE_PREEMPT_IF_NEEDED();
7102 HMVMX_ASSERT_CPU_SAFE();
7103 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7104 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7105
7106 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7107 and done this from the VMXR0ThreadCtxCallback(). */
7108 if (!pVCpu->hm.s.fLeaveDone)
7109 {
7110 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
7111 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
7112 pVCpu->hm.s.fLeaveDone = true;
7113 }
7114
7115 /*
7116 * !!! IMPORTANT !!!
7117 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7118 */
7119
7120 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7121 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
7122 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7123 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7124 VMMR0ThreadCtxHooksDeregister(pVCpu);
7125
7126 /* Leave HM context. This takes care of local init (term). */
7127 int rc = HMR0LeaveCpu(pVCpu);
7128
7129 HM_RESTORE_PREEMPT_IF_NEEDED();
7130
7131 return rc;
7132}
7133
7134
7135/**
7136 * Does the necessary state syncing before doing a longjmp to ring-3.
7137 *
7138 * @returns VBox status code.
7139 * @param pVM Pointer to the VM.
7140 * @param pVCpu Pointer to the VMCPU.
7141 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7142 * out-of-sync. Make sure to update the required fields
7143 * before using them.
7144 *
7145 * @remarks No-long-jmp zone!!!
7146 */
7147DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7148{
7149 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7150}
7151
7152
7153/**
7154 * Take necessary actions before going back to ring-3.
7155 *
7156 * An action requires us to go back to ring-3. This function does the necessary
7157 * steps before we can safely return to ring-3. This is not the same as longjmps
7158 * to ring-3, this is voluntary and prepares the guest so it may continue
7159 * executing outside HM (recompiler/IEM).
7160 *
7161 * @returns VBox status code.
7162 * @param pVM Pointer to the VM.
7163 * @param pVCpu Pointer to the VMCPU.
7164 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7165 * out-of-sync. Make sure to update the required fields
7166 * before using them.
7167 * @param rcExit The reason for exiting to ring-3. Can be
7168 * VINF_VMM_UNKNOWN_RING3_CALL.
7169 */
7170static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7171{
7172 Assert(pVM);
7173 Assert(pVCpu);
7174 Assert(pMixedCtx);
7175 HMVMX_ASSERT_PREEMPT_SAFE();
7176
7177 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7178 {
7179 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7180 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7181 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7182 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7183 }
7184
7185 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7186 VMMRZCallRing3Disable(pVCpu);
7187 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7188
7189 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7190 if (pVCpu->hm.s.Event.fPending)
7191 {
7192 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7193 Assert(!pVCpu->hm.s.Event.fPending);
7194 }
7195
7196 /* Save guest state and restore host state bits. */
7197 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7198 AssertRCReturn(rc, rc);
7199 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7200
7201 /* Sync recompiler state. */
7202 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7203 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7204 | CPUM_CHANGED_LDTR
7205 | CPUM_CHANGED_GDTR
7206 | CPUM_CHANGED_IDTR
7207 | CPUM_CHANGED_TR
7208 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7209 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7210 if ( pVM->hm.s.fNestedPaging
7211 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7212 {
7213 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7214 }
7215
7216 Assert(!pVCpu->hm.s.fClearTrapFlag);
7217
7218 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7219 if (rcExit != VINF_EM_RAW_INTERRUPT)
7220 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7221
7222 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7223
7224 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7225 VMMRZCallRing3RemoveNotification(pVCpu);
7226 VMMRZCallRing3Enable(pVCpu);
7227
7228 return rc;
7229}
7230
7231
7232/**
7233 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7234 * longjump to ring-3 and possibly get preempted.
7235 *
7236 * @returns VBox status code.
7237 * @param pVCpu Pointer to the VMCPU.
7238 * @param enmOperation The operation causing the ring-3 longjump.
7239 * @param pvUser Opaque pointer to the guest-CPU context. The data
7240 * may be out-of-sync. Make sure to update the required
7241 * fields before using them.
7242 */
7243DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7244{
7245 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7246 {
7247 /*
7248 * !!! IMPORTANT !!!
7249 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7250 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7251 */
7252 VMMRZCallRing3RemoveNotification(pVCpu);
7253 VMMRZCallRing3Disable(pVCpu);
7254 HM_DISABLE_PREEMPT_IF_NEEDED();
7255
7256 PVM pVM = pVCpu->CTX_SUFF(pVM);
7257 if (CPUMIsGuestFPUStateActive(pVCpu))
7258 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7259
7260 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7261
7262#if HC_ARCH_BITS == 64
7263 /* Restore host-state bits that VT-x only restores partially. */
7264 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7265 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7266 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7267 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7268
7269 /* Restore the host MSRs as we're leaving VT-x context. */
7270 if ( pVM->hm.s.fAllow64BitGuests
7271 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7272 {
7273 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7274 }
7275#endif
7276 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7277 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7278 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7279 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7280 {
7281 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7282 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7283 }
7284
7285 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7286 VMMR0ThreadCtxHooksDeregister(pVCpu);
7287
7288 HMR0LeaveCpu(pVCpu);
7289 HM_RESTORE_PREEMPT_IF_NEEDED();
7290 return VINF_SUCCESS;
7291 }
7292
7293 Assert(pVCpu);
7294 Assert(pvUser);
7295 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7296 HMVMX_ASSERT_PREEMPT_SAFE();
7297
7298 VMMRZCallRing3Disable(pVCpu);
7299 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7300
7301 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7302 enmOperation));
7303
7304 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7305 AssertRCReturn(rc, rc);
7306
7307 VMMRZCallRing3Enable(pVCpu);
7308 return VINF_SUCCESS;
7309}
7310
7311
7312/**
7313 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7314 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7315 *
7316 * @param pVCpu Pointer to the VMCPU.
7317 */
7318DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7319{
7320 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7321 {
7322 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7323 {
7324 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7325 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7326 AssertRC(rc);
7327 Log4(("Setup interrupt-window exiting\n"));
7328 }
7329 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7330}
7331
7332
7333/**
7334 * Clears the interrupt-window exiting control in the VMCS.
7335 *
7336 * @param pVCpu Pointer to the VMCPU.
7337 */
7338DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7339{
7340 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7341 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7342 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7343 AssertRC(rc);
7344 Log4(("Cleared interrupt-window exiting\n"));
7345}
7346
7347
7348/**
7349 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7350 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7351 *
7352 * @param pVCpu Pointer to the VMCPU.
7353 */
7354DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7355{
7356 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7357 {
7358 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7359 {
7360 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7361 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7362 AssertRC(rc);
7363 Log4(("Setup NMI-window exiting\n"));
7364 }
7365 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7366}
7367
7368
7369/**
7370 * Clears the NMI-window exiting control in the VMCS.
7371 *
7372 * @param pVCpu Pointer to the VMCPU.
7373 */
7374DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7375{
7376 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7377 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7378 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7379 AssertRC(rc);
7380 Log4(("Cleared NMI-window exiting\n"));
7381}
7382
7383
7384/**
7385 * Evaluates the event to be delivered to the guest and sets it as the pending
7386 * event.
7387 *
7388 * @param pVCpu Pointer to the VMCPU.
7389 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7390 * out-of-sync. Make sure to update the required fields
7391 * before using them.
7392 */
7393static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7394{
7395 Assert(!pVCpu->hm.s.Event.fPending);
7396
7397 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7398 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7399 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7400 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7401 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7402
7403 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7404 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7405 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7406 Assert(!TRPMHasTrap(pVCpu));
7407
7408 /** @todo SMI. SMIs take priority over NMIs. */
7409 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7410 {
7411 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7412 if ( !fBlockNmi
7413 && !fBlockSti
7414 && !fBlockMovSS)
7415 {
7416 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7417 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7418 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7419
7420 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7421 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7422 }
7423 else
7424 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7425 }
7426 /*
7427 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7428 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7429 */
7430 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7431 && !pVCpu->hm.s.fSingleInstruction)
7432 {
7433 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7434 AssertRC(rc);
7435 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7436 if ( !fBlockInt
7437 && !fBlockSti
7438 && !fBlockMovSS)
7439 {
7440 uint8_t u8Interrupt;
7441 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7442 if (RT_SUCCESS(rc))
7443 {
7444 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7445 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7446 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7447
7448 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7449 }
7450 else
7451 {
7452 /** @todo Does this actually happen? If not turn it into an assertion. */
7453 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7454 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7455 }
7456 }
7457 else
7458 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7459 }
7460}
7461
7462
7463/**
7464 * Sets a pending-debug exception to be delivered to the guest if the guest is
7465 * single-stepping.
7466 *
7467 * @param pVCpu Pointer to the VMCPU.
7468 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7469 * out-of-sync. Make sure to update the required fields
7470 * before using them.
7471 */
7472DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7473{
7474 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7475 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7476 {
7477 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7478 AssertRC(rc);
7479 }
7480}
7481
7482
7483/**
7484 * Injects any pending events into the guest if the guest is in a state to
7485 * receive them.
7486 *
7487 * @returns VBox status code (informational status codes included).
7488 * @param pVCpu Pointer to the VMCPU.
7489 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7490 * out-of-sync. Make sure to update the required fields
7491 * before using them.
7492 */
7493static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7494{
7495 HMVMX_ASSERT_PREEMPT_SAFE();
7496 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7497
7498 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7499 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7500 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7501 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7502
7503 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7504 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7505 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7506 Assert(!TRPMHasTrap(pVCpu));
7507
7508 int rc = VINF_SUCCESS;
7509 if (pVCpu->hm.s.Event.fPending)
7510 {
7511 /*
7512 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7513 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7514 * ended up enabling interrupts outside VT-x.
7515 */
7516 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7517 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7518 && uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7519 {
7520 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7521 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7522 }
7523
7524#ifdef VBOX_STRICT
7525 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7526 {
7527 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7528 if (fBlockInt)
7529 return VERR_VMX_IPE_4;
7530 Assert(!fBlockSti);
7531 Assert(!fBlockMovSS);
7532 }
7533 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7534 {
7535 bool fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7536 Assert(!fBlockSti);
7537 Assert(!fBlockMovSS);
7538 Assert(!fBlockNmi);
7539 }
7540#endif
7541 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7542 (uint8_t)uIntType));
7543 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7544 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7545 AssertRCReturn(rc, rc);
7546
7547 /* Update the interruptibility-state as it could have been changed by
7548 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7549 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7550 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7551
7552#ifdef VBOX_WITH_STATISTICS
7553 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7554 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7555 else
7556 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7557#endif
7558 }
7559
7560 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7561 if ( fBlockSti
7562 || fBlockMovSS)
7563 {
7564 if ( !pVCpu->hm.s.fSingleInstruction
7565 && !DBGFIsStepping(pVCpu))
7566 {
7567 /*
7568 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7569 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7570 * See Intel spec. 27.3.4 "Saving Non-Register State".
7571 */
7572 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7573 AssertRCReturn(rc2, rc2);
7574 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7575 }
7576 else if (pMixedCtx->eflags.Bits.u1TF)
7577 {
7578 /*
7579 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7580 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7581 */
7582 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7583 uIntrState = 0;
7584 }
7585 }
7586
7587 /*
7588 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7589 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7590 */
7591 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7592 AssertRC(rc2);
7593
7594 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7595 NOREF(fBlockMovSS); NOREF(fBlockSti);
7596 return rc;
7597}
7598
7599
7600/**
7601 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7602 *
7603 * @param pVCpu Pointer to the VMCPU.
7604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7605 * out-of-sync. Make sure to update the required fields
7606 * before using them.
7607 */
7608DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7609{
7610 NOREF(pMixedCtx);
7611 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7612 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7613}
7614
7615
7616/**
7617 * Injects a double-fault (#DF) exception into the VM.
7618 *
7619 * @returns VBox status code (informational status code included).
7620 * @param pVCpu Pointer to the VMCPU.
7621 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7622 * out-of-sync. Make sure to update the required fields
7623 * before using them.
7624 */
7625DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7626{
7627 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7628 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7629 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7630 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7631 puIntrState);
7632}
7633
7634
7635/**
7636 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7637 *
7638 * @param pVCpu Pointer to the VMCPU.
7639 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7640 * out-of-sync. Make sure to update the required fields
7641 * before using them.
7642 */
7643DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7644{
7645 NOREF(pMixedCtx);
7646 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7648 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7649}
7650
7651
7652/**
7653 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7654 *
7655 * @param pVCpu Pointer to the VMCPU.
7656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7657 * out-of-sync. Make sure to update the required fields
7658 * before using them.
7659 * @param cbInstr The value of RIP that is to be pushed on the guest
7660 * stack.
7661 */
7662DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7663{
7664 NOREF(pMixedCtx);
7665 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7666 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7667 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7668}
7669
7670
7671/**
7672 * Injects a general-protection (#GP) fault into the VM.
7673 *
7674 * @returns VBox status code (informational status code included).
7675 * @param pVCpu Pointer to the VMCPU.
7676 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7677 * out-of-sync. Make sure to update the required fields
7678 * before using them.
7679 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7680 * mode, i.e. in real-mode it's not valid).
7681 * @param u32ErrorCode The error code associated with the #GP.
7682 */
7683DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7684 uint32_t *puIntrState)
7685{
7686 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7688 if (fErrorCodeValid)
7689 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7690 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7691 puIntrState);
7692}
7693
7694
7695/**
7696 * Sets a general-protection (#GP) exception as pending-for-injection into the
7697 * VM.
7698 *
7699 * @param pVCpu Pointer to the VMCPU.
7700 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7701 * out-of-sync. Make sure to update the required fields
7702 * before using them.
7703 * @param u32ErrorCode The error code associated with the #GP.
7704 */
7705DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7706{
7707 NOREF(pMixedCtx);
7708 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7709 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7710 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7711 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7712}
7713
7714
7715/**
7716 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7717 *
7718 * @param pVCpu Pointer to the VMCPU.
7719 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7720 * out-of-sync. Make sure to update the required fields
7721 * before using them.
7722 * @param uVector The software interrupt vector number.
7723 * @param cbInstr The value of RIP that is to be pushed on the guest
7724 * stack.
7725 */
7726DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7727{
7728 NOREF(pMixedCtx);
7729 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7730 if ( uVector == X86_XCPT_BP
7731 || uVector == X86_XCPT_OF)
7732 {
7733 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7734 }
7735 else
7736 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7737 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7738}
7739
7740
7741/**
7742 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7743 * stack.
7744 *
7745 * @returns VBox status code (information status code included).
7746 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7747 * @param pVM Pointer to the VM.
7748 * @param pMixedCtx Pointer to the guest-CPU context.
7749 * @param uValue The value to push to the guest stack.
7750 */
7751DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7752{
7753 /*
7754 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7755 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7756 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7757 */
7758 if (pMixedCtx->sp == 1)
7759 return VINF_EM_RESET;
7760 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7761 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7762 AssertRCReturn(rc, rc);
7763 return rc;
7764}
7765
7766
7767/**
7768 * Injects an event into the guest upon VM-entry by updating the relevant fields
7769 * in the VM-entry area in the VMCS.
7770 *
7771 * @returns VBox status code (informational error codes included).
7772 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7773 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7774 *
7775 * @param pVCpu Pointer to the VMCPU.
7776 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7777 * be out-of-sync. Make sure to update the required
7778 * fields before using them.
7779 * @param u64IntInfo The VM-entry interruption-information field.
7780 * @param cbInstr The VM-entry instruction length in bytes (for
7781 * software interrupts, exceptions and privileged
7782 * software exceptions).
7783 * @param u32ErrCode The VM-entry exception error code.
7784 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7785 * @param puIntrState Pointer to the current guest interruptibility-state.
7786 * This interruptibility-state will be updated if
7787 * necessary. This cannot not be NULL.
7788 *
7789 * @remarks Requires CR0!
7790 * @remarks No-long-jump zone!!!
7791 */
7792static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7793 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7794{
7795 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7796 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7797 Assert(puIntrState);
7798 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7799
7800 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7801 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7802
7803#ifdef VBOX_STRICT
7804 /* Validate the error-code-valid bit for hardware exceptions. */
7805 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7806 {
7807 switch (uVector)
7808 {
7809 case X86_XCPT_PF:
7810 case X86_XCPT_DF:
7811 case X86_XCPT_TS:
7812 case X86_XCPT_NP:
7813 case X86_XCPT_SS:
7814 case X86_XCPT_GP:
7815 case X86_XCPT_AC:
7816 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7817 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7818 /* fallthru */
7819 default:
7820 break;
7821 }
7822 }
7823#endif
7824
7825 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7826 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7827 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7828
7829 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7830
7831 /* We require CR0 to check if the guest is in real-mode. */
7832 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7833 AssertRCReturn(rc, rc);
7834
7835 /*
7836 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7837 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7838 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7839 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7840 */
7841 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7842 {
7843 PVM pVM = pVCpu->CTX_SUFF(pVM);
7844 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7845 {
7846 Assert(PDMVmmDevHeapIsEnabled(pVM));
7847 Assert(pVM->hm.s.vmx.pRealModeTSS);
7848
7849 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7850 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7851 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7852 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7853 AssertRCReturn(rc, rc);
7854 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7855
7856 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7857 size_t const cbIdtEntry = sizeof(X86IDTR16);
7858 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7859 {
7860 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7861 if (uVector == X86_XCPT_DF)
7862 return VINF_EM_RESET;
7863 else if (uVector == X86_XCPT_GP)
7864 {
7865 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7866 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7867 }
7868
7869 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7870 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7871 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7872 }
7873
7874 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7875 uint16_t uGuestIp = pMixedCtx->ip;
7876 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7877 {
7878 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7879 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7880 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7881 }
7882 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7883 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7884
7885 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7886 X86IDTR16 IdtEntry;
7887 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7888 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7889 AssertRCReturn(rc, rc);
7890
7891 /* Construct the stack frame for the interrupt/exception handler. */
7892 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7893 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7894 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7895 AssertRCReturn(rc, rc);
7896
7897 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7898 if (rc == VINF_SUCCESS)
7899 {
7900 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7901 pMixedCtx->rip = IdtEntry.offSel;
7902 pMixedCtx->cs.Sel = IdtEntry.uSel;
7903 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7904 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7905 && uVector == X86_XCPT_PF)
7906 {
7907 pMixedCtx->cr2 = GCPtrFaultAddress;
7908 }
7909
7910 /* If any other guest-state bits are changed here, make sure to update
7911 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7912 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7913 | HM_CHANGED_GUEST_RIP
7914 | HM_CHANGED_GUEST_RFLAGS
7915 | HM_CHANGED_GUEST_RSP);
7916
7917 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7918 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7919 {
7920 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7921 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7922 Log4(("Clearing inhibition due to STI.\n"));
7923 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7924 }
7925 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7926
7927 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7928 it, if we are returning to ring-3 before executing guest code. */
7929 pVCpu->hm.s.Event.fPending = false;
7930 }
7931 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7932 return rc;
7933 }
7934 else
7935 {
7936 /*
7937 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7938 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7939 */
7940 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7941 }
7942 }
7943
7944 /* Validate. */
7945 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7946 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7947 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7948
7949 /* Inject. */
7950 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7951 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7952 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7953 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7954
7955 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7956 && uVector == X86_XCPT_PF)
7957 {
7958 pMixedCtx->cr2 = GCPtrFaultAddress;
7959 }
7960
7961 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7962 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7963
7964 AssertRCReturn(rc, rc);
7965 return rc;
7966}
7967
7968
7969/**
7970 * Clears the interrupt-window exiting control in the VMCS and if necessary
7971 * clears the current event in the VMCS as well.
7972 *
7973 * @returns VBox status code.
7974 * @param pVCpu Pointer to the VMCPU.
7975 *
7976 * @remarks Use this function only to clear events that have not yet been
7977 * delivered to the guest but are injected in the VMCS!
7978 * @remarks No-long-jump zone!!!
7979 */
7980static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7981{
7982 int rc;
7983 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7984
7985 /* Clear interrupt-window exiting control. */
7986 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7987 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7988
7989 if (!pVCpu->hm.s.Event.fPending)
7990 return;
7991
7992#ifdef VBOX_STRICT
7993 uint32_t u32EntryInfo;
7994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7995 AssertRC(rc);
7996 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7997#endif
7998
7999 /* Clear the entry-interruption field (including the valid bit). */
8000 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
8001 AssertRC(rc);
8002
8003 /* Clear the pending debug exception field. */
8004 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
8005 AssertRC(rc);
8006
8007 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
8008 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
8009}
8010
8011
8012/**
8013 * Enters the VT-x session.
8014 *
8015 * @returns VBox status code.
8016 * @param pVM Pointer to the VM.
8017 * @param pVCpu Pointer to the VMCPU.
8018 * @param pCpu Pointer to the CPU info struct.
8019 */
8020VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8021{
8022 AssertPtr(pVM);
8023 AssertPtr(pVCpu);
8024 Assert(pVM->hm.s.vmx.fSupported);
8025 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8026 NOREF(pCpu); NOREF(pVM);
8027
8028 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8029 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8030
8031#ifdef VBOX_STRICT
8032 /* Make sure we're in VMX root mode. */
8033 RTCCUINTREG u32HostCR4 = ASMGetCR4();
8034 if (!(u32HostCR4 & X86_CR4_VMXE))
8035 {
8036 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8037 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8038 }
8039#endif
8040
8041 /*
8042 * Load the VCPU's VMCS as the current (and active) one.
8043 */
8044 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8045 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8046 if (RT_FAILURE(rc))
8047 return rc;
8048
8049 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8050 pVCpu->hm.s.fLeaveDone = false;
8051 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8052
8053 return VINF_SUCCESS;
8054}
8055
8056
8057/**
8058 * The thread-context callback (only on platforms which support it).
8059 *
8060 * @param enmEvent The thread-context event.
8061 * @param pVCpu Pointer to the VMCPU.
8062 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8063 * @thread EMT(pVCpu)
8064 */
8065VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8066{
8067 NOREF(fGlobalInit);
8068
8069 switch (enmEvent)
8070 {
8071 case RTTHREADCTXEVENT_PREEMPTING:
8072 {
8073 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8074 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8075 VMCPU_ASSERT_EMT(pVCpu);
8076
8077 PVM pVM = pVCpu->CTX_SUFF(pVM);
8078 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8079
8080 /* No longjmps (logger flushes, locks) in this fragile context. */
8081 VMMRZCallRing3Disable(pVCpu);
8082 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8083
8084 /*
8085 * Restore host-state (FPU, debug etc.)
8086 */
8087 if (!pVCpu->hm.s.fLeaveDone)
8088 {
8089 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8090 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8091 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
8092 pVCpu->hm.s.fLeaveDone = true;
8093 }
8094
8095 /* Leave HM context, takes care of local init (term). */
8096 int rc = HMR0LeaveCpu(pVCpu);
8097 AssertRC(rc); NOREF(rc);
8098
8099 /* Restore longjmp state. */
8100 VMMRZCallRing3Enable(pVCpu);
8101 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
8102 break;
8103 }
8104
8105 case RTTHREADCTXEVENT_RESUMED:
8106 {
8107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8108 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
8109 VMCPU_ASSERT_EMT(pVCpu);
8110
8111 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8112 VMMRZCallRing3Disable(pVCpu);
8113 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8114
8115 /* Initialize the bare minimum state required for HM. This takes care of
8116 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8117 int rc = HMR0EnterCpu(pVCpu);
8118 AssertRC(rc);
8119 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8120
8121 /* Load the active VMCS as the current one. */
8122 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8123 {
8124 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8125 AssertRC(rc); NOREF(rc);
8126 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8127 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8128 }
8129 pVCpu->hm.s.fLeaveDone = false;
8130
8131 /* Restore longjmp state. */
8132 VMMRZCallRing3Enable(pVCpu);
8133 break;
8134 }
8135
8136 default:
8137 break;
8138 }
8139}
8140
8141
8142/**
8143 * Saves the host state in the VMCS host-state.
8144 * Sets up the VM-exit MSR-load area.
8145 *
8146 * The CPU state will be loaded from these fields on every successful VM-exit.
8147 *
8148 * @returns VBox status code.
8149 * @param pVM Pointer to the VM.
8150 * @param pVCpu Pointer to the VMCPU.
8151 *
8152 * @remarks No-long-jump zone!!!
8153 */
8154static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8155{
8156 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8157
8158 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8159 return VINF_SUCCESS;
8160
8161 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8162 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8163
8164 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8165 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8166
8167 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8168 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8169
8170 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8171 return rc;
8172}
8173
8174
8175/**
8176 * Saves the host state in the VMCS host-state.
8177 *
8178 * @returns VBox status code.
8179 * @param pVM Pointer to the VM.
8180 * @param pVCpu Pointer to the VMCPU.
8181 *
8182 * @remarks No-long-jump zone!!!
8183 */
8184VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8185{
8186 AssertPtr(pVM);
8187 AssertPtr(pVCpu);
8188
8189 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8190
8191 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8192 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8193 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8194 return hmR0VmxSaveHostState(pVM, pVCpu);
8195}
8196
8197
8198/**
8199 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8200 * loaded from these fields on every successful VM-entry.
8201 *
8202 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8203 * Sets up the VM-entry controls.
8204 * Sets up the appropriate VMX non-root function to execute guest code based on
8205 * the guest CPU mode.
8206 *
8207 * @returns VBox status code.
8208 * @param pVM Pointer to the VM.
8209 * @param pVCpu Pointer to the VMCPU.
8210 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8211 * out-of-sync. Make sure to update the required fields
8212 * before using them.
8213 *
8214 * @remarks No-long-jump zone!!!
8215 */
8216static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8217{
8218 AssertPtr(pVM);
8219 AssertPtr(pVCpu);
8220 AssertPtr(pMixedCtx);
8221 HMVMX_ASSERT_PREEMPT_SAFE();
8222
8223#ifdef LOG_ENABLED
8224 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8225 * probably not initialized yet? Anyway this will do for now.
8226 *
8227 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8228 * interface and disable ring-3 calls when thread-context hooks are not
8229 * available. */
8230 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8231 VMMR0LogFlushDisable(pVCpu);
8232#endif
8233
8234 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8235
8236 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8237
8238 /* Determine real-on-v86 mode. */
8239 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8240 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8241 && CPUMIsGuestInRealModeEx(pMixedCtx))
8242 {
8243 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8244 }
8245
8246 /*
8247 * Load the guest-state into the VMCS.
8248 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8249 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8250 */
8251 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8252 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8253
8254 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8255 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8256 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8257
8258 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8259 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8260 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8261
8262 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8263 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8264
8265 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8266 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8267
8268 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8269 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8270 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8271
8272 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8273 determine we don't have to swap EFER after all. */
8274 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8275 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8276
8277 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8278 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8279
8280 /*
8281 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8282 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8283 */
8284 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8285 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8286
8287 /* Clear any unused and reserved bits. */
8288 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8289
8290#ifdef LOG_ENABLED
8291 /* Only reenable log-flushing if the caller has it enabled. */
8292 if (!fCallerDisabledLogFlush)
8293 VMMR0LogFlushEnable(pVCpu);
8294#endif
8295
8296 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8297 return rc;
8298}
8299
8300
8301/**
8302 * Loads the state shared between the host and guest into the VMCS.
8303 *
8304 * @param pVM Pointer to the VM.
8305 * @param pVCpu Pointer to the VMCPU.
8306 * @param pCtx Pointer to the guest-CPU context.
8307 *
8308 * @remarks No-long-jump zone!!!
8309 */
8310static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8311{
8312 NOREF(pVM);
8313
8314 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8315 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8316
8317 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8318 {
8319 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8320 AssertRC(rc);
8321 }
8322
8323 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8324 {
8325 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8326 AssertRC(rc);
8327
8328 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8329 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8330 {
8331 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8332 AssertRC(rc);
8333 }
8334 }
8335
8336 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8337 {
8338#if HC_ARCH_BITS == 64
8339 if (pVM->hm.s.fAllow64BitGuests)
8340 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8341#endif
8342 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8343 }
8344
8345 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8346 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8347}
8348
8349
8350/**
8351 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8352 *
8353 * @param pVM Pointer to the VM.
8354 * @param pVCpu Pointer to the VMCPU.
8355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8356 * out-of-sync. Make sure to update the required fields
8357 * before using them.
8358 */
8359DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8360{
8361 HMVMX_ASSERT_PREEMPT_SAFE();
8362
8363 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8364#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8365 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8366#endif
8367
8368 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8369 {
8370 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8371 AssertRC(rc);
8372 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8373 }
8374 else if (HMCPU_CF_VALUE(pVCpu))
8375 {
8376 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8377 AssertRC(rc);
8378 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8379 }
8380
8381 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8382 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8383 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8384 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8385}
8386
8387
8388/**
8389 * Does the preparations before executing guest code in VT-x.
8390 *
8391 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8392 * recompiler. We must be cautious what we do here regarding committing
8393 * guest-state information into the VMCS assuming we assuredly execute the
8394 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8395 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8396 * so that the recompiler can (and should) use them when it resumes guest
8397 * execution. Otherwise such operations must be done when we can no longer
8398 * exit to ring-3.
8399 *
8400 * @returns Strict VBox status code.
8401 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8402 * have been disabled.
8403 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8404 * double-fault into the guest.
8405 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8406 *
8407 * @param pVM Pointer to the VM.
8408 * @param pVCpu Pointer to the VMCPU.
8409 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8410 * out-of-sync. Make sure to update the required fields
8411 * before using them.
8412 * @param pVmxTransient Pointer to the VMX transient structure.
8413 */
8414static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8415{
8416 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8417
8418#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8419 PGMRZDynMapFlushAutoSet(pVCpu);
8420#endif
8421
8422 /* Check force flag actions that might require us to go back to ring-3. */
8423 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8424 if (rc != VINF_SUCCESS)
8425 return rc;
8426
8427#ifndef IEM_VERIFICATION_MODE_FULL
8428 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8429 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8430 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8431 {
8432 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8433 RTGCPHYS GCPhysApicBase;
8434 GCPhysApicBase = pMixedCtx->msrApicBase;
8435 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8436
8437 /* Unalias any existing mapping. */
8438 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8439 AssertRCReturn(rc, rc);
8440
8441 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8442 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8443 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8444 AssertRCReturn(rc, rc);
8445
8446 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8447 }
8448#endif /* !IEM_VERIFICATION_MODE_FULL */
8449
8450 /*
8451 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8452 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8453 */
8454 if (TRPMHasTrap(pVCpu))
8455 {
8456 rc = hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8457 if (RT_FAILURE(rc))
8458 return rc;
8459 }
8460 else if (!pVCpu->hm.s.Event.fPending)
8461 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8462
8463 /*
8464 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8465 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8466 */
8467 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8468 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8469 {
8470 //Assert(rc == VINF_EM_RESET);
8471 return rc;
8472 }
8473
8474 /*
8475 * Load the guest state bits, we can handle longjmps/getting preempted here.
8476 *
8477 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8478 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8479 * Hence, this needs to be done -after- injection of events.
8480 */
8481 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8482
8483 /*
8484 * No longjmps to ring-3 from this point on!!!
8485 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8486 * This also disables flushing of the R0-logger instance (if any).
8487 */
8488 VMMRZCallRing3Disable(pVCpu);
8489
8490 /*
8491 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8492 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8493 *
8494 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8495 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8496 *
8497 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8498 * executing guest code.
8499 */
8500 pVmxTransient->uEflags = ASMIntDisableFlags();
8501 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8502 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8503 {
8504 hmR0VmxClearEventVmcs(pVCpu);
8505 ASMSetFlags(pVmxTransient->uEflags);
8506 VMMRZCallRing3Enable(pVCpu);
8507 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8508 return VINF_EM_RAW_TO_R3;
8509 }
8510
8511 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8512 {
8513 hmR0VmxClearEventVmcs(pVCpu);
8514 ASMSetFlags(pVmxTransient->uEflags);
8515 VMMRZCallRing3Enable(pVCpu);
8516 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8517 return VINF_EM_RAW_INTERRUPT;
8518 }
8519
8520 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8521 pVCpu->hm.s.Event.fPending = false;
8522
8523 return VINF_SUCCESS;
8524}
8525
8526
8527/**
8528 * Prepares to run guest code in VT-x and we've committed to doing so. This
8529 * means there is no backing out to ring-3 or anywhere else at this
8530 * point.
8531 *
8532 * @param pVM Pointer to the VM.
8533 * @param pVCpu Pointer to the VMCPU.
8534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8535 * out-of-sync. Make sure to update the required fields
8536 * before using them.
8537 * @param pVmxTransient Pointer to the VMX transient structure.
8538 *
8539 * @remarks Called with preemption disabled.
8540 * @remarks No-long-jump zone!!!
8541 */
8542static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8543{
8544 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8545 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8546 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8547
8548 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8549 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8550
8551#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8552 if (!CPUMIsGuestFPUStateActive(pVCpu))
8553 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8554 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8555#endif
8556
8557 if ( pVCpu->hm.s.fUseGuestFpu
8558 && !CPUMIsGuestFPUStateActive(pVCpu))
8559 {
8560 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8561 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8562 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8563 }
8564
8565 /*
8566 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8567 */
8568 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8569 && pVCpu->hm.s.vmx.cMsrs > 0)
8570 {
8571 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8572 }
8573
8574 /*
8575 * Load the host state bits as we may've been preempted (only happens when
8576 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8577 */
8578 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8579 {
8580 /* This ASSUMES that pfnStartVM has been set up already. */
8581 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8582 AssertRC(rc);
8583 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8584 }
8585 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8586
8587 /*
8588 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8589 */
8590 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8591 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8592 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8593
8594 /* Store status of the shared guest-host state at the time of VM-entry. */
8595#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8596 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8597 {
8598 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8599 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8600 }
8601 else
8602#endif
8603 {
8604 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8605 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8606 }
8607 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8608
8609 /*
8610 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8611 */
8612 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8613 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8614
8615 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8616 RTCPUID idCurrentCpu = pCpu->idCpu;
8617 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8618 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8619 {
8620 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8621 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8622 }
8623
8624 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8625 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8626 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8627 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8628
8629 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8630
8631 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8632 to start executing. */
8633
8634 /*
8635 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8636 */
8637 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8638 {
8639 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8640 {
8641 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8642 AssertRC(rc2);
8643 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8644 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8645 true /* fUpdateHostMsr */);
8646 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8647 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8648 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8649 }
8650 else
8651 {
8652 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8653 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8654 }
8655 }
8656
8657#ifdef VBOX_STRICT
8658 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8659 hmR0VmxCheckHostEferMsr(pVCpu);
8660 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8661#endif
8662#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8663 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8664 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8665 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8666#endif
8667}
8668
8669
8670/**
8671 * Performs some essential restoration of state after running guest code in
8672 * VT-x.
8673 *
8674 * @param pVM Pointer to the VM.
8675 * @param pVCpu Pointer to the VMCPU.
8676 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8677 * out-of-sync. Make sure to update the required fields
8678 * before using them.
8679 * @param pVmxTransient Pointer to the VMX transient structure.
8680 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8681 *
8682 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8683 *
8684 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8685 * unconditionally when it is safe to do so.
8686 */
8687static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8688{
8689 NOREF(pVM);
8690
8691 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8692
8693 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8694 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8695 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8696 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8697 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8698
8699 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8700 {
8701 /** @todo Find a way to fix hardcoding a guestimate. */
8702 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8703 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8704 }
8705
8706 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8707 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8708 Assert(!(ASMGetFlags() & X86_EFL_IF));
8709 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8710
8711#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8712 if (CPUMIsGuestFPUStateActive(pVCpu))
8713 {
8714 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8715 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8716 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8717 }
8718#endif
8719
8720#if HC_ARCH_BITS == 64
8721 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8722#endif
8723 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8724#ifdef VBOX_STRICT
8725 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8726#endif
8727 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8728 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8729
8730 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8731 uint32_t uExitReason;
8732 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8733 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8734 AssertRC(rc);
8735 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8736 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8737
8738 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8739 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8740 {
8741 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8742 pVmxTransient->fVMEntryFailed));
8743 return;
8744 }
8745
8746 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8747 {
8748 /* Update the guest interruptibility-state from the VMCS. */
8749 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8750
8751#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8752 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8753 AssertRC(rc);
8754#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8755 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8756 AssertRC(rc);
8757#endif
8758
8759 /*
8760 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8761 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8762 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8763 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8764 */
8765 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8766 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8767 {
8768 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8769 AssertRC(rc);
8770 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8771 }
8772 }
8773}
8774
8775
8776/**
8777 * Runs the guest code using VT-x the normal way.
8778 *
8779 * @returns VBox status code.
8780 * @param pVM Pointer to the VM.
8781 * @param pVCpu Pointer to the VMCPU.
8782 * @param pCtx Pointer to the guest-CPU context.
8783 *
8784 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8785 */
8786static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8787{
8788 VMXTRANSIENT VmxTransient;
8789 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8790 int rc = VERR_INTERNAL_ERROR_5;
8791 uint32_t cLoops = 0;
8792
8793 for (;; cLoops++)
8794 {
8795 Assert(!HMR0SuspendPending());
8796 HMVMX_ASSERT_CPU_SAFE();
8797
8798 /* Preparatory work for running guest code, this may force us to return
8799 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8800 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8801 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8802 if (rc != VINF_SUCCESS)
8803 break;
8804
8805 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8806 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8807 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8808
8809 /* Restore any residual host-state and save any bits shared between host
8810 and guest into the guest-CPU state. Re-enables interrupts! */
8811 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8812
8813 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8814 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8815 {
8816 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8817 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8818 return rc;
8819 }
8820
8821 /* Handle the VM-exit. */
8822 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8824 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8825 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8826 HMVMX_START_EXIT_DISPATCH_PROF();
8827#ifdef HMVMX_USE_FUNCTION_TABLE
8828 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8829#else
8830 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8831#endif
8832 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8833 if (rc != VINF_SUCCESS)
8834 break;
8835 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8836 {
8837 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8838 rc = VINF_EM_RAW_INTERRUPT;
8839 break;
8840 }
8841 }
8842
8843 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8844 return rc;
8845}
8846
8847
8848/**
8849 * Single steps guest code using VT-x.
8850 *
8851 * @returns VBox status code.
8852 * @param pVM Pointer to the VM.
8853 * @param pVCpu Pointer to the VMCPU.
8854 * @param pCtx Pointer to the guest-CPU context.
8855 *
8856 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8857 */
8858static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8859{
8860 VMXTRANSIENT VmxTransient;
8861 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8862 int rc = VERR_INTERNAL_ERROR_5;
8863 uint32_t cLoops = 0;
8864 uint16_t uCsStart = pCtx->cs.Sel;
8865 uint64_t uRipStart = pCtx->rip;
8866
8867 for (;; cLoops++)
8868 {
8869 Assert(!HMR0SuspendPending());
8870 HMVMX_ASSERT_CPU_SAFE();
8871
8872 /* Preparatory work for running guest code, this may force us to return
8873 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8874 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8875 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8876 if (rc != VINF_SUCCESS)
8877 break;
8878
8879 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8880 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8881 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8882
8883 /* Restore any residual host-state and save any bits shared between host
8884 and guest into the guest-CPU state. Re-enables interrupts! */
8885 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8886
8887 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8888 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8889 {
8890 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8891 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8892 return rc;
8893 }
8894
8895 /* Handle the VM-exit. */
8896 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8897 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8898 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8899 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8900 HMVMX_START_EXIT_DISPATCH_PROF();
8901#ifdef HMVMX_USE_FUNCTION_TABLE
8902 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8903#else
8904 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8905#endif
8906 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8907 if (rc != VINF_SUCCESS)
8908 break;
8909 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8910 {
8911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8912 rc = VINF_EM_RAW_INTERRUPT;
8913 break;
8914 }
8915
8916 /*
8917 * Did the RIP change, if so, consider it a single step.
8918 * Otherwise, make sure one of the TFs gets set.
8919 */
8920 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8921 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8922 AssertRCReturn(rc2, rc2);
8923 if ( pCtx->rip != uRipStart
8924 || pCtx->cs.Sel != uCsStart)
8925 {
8926 rc = VINF_EM_DBG_STEPPED;
8927 break;
8928 }
8929 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8930 }
8931
8932 /*
8933 * Clear the X86_EFL_TF if necessary.
8934 */
8935 if (pVCpu->hm.s.fClearTrapFlag)
8936 {
8937 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8938 AssertRCReturn(rc2, rc2);
8939 pVCpu->hm.s.fClearTrapFlag = false;
8940 pCtx->eflags.Bits.u1TF = 0;
8941 }
8942 /** @todo there seems to be issues with the resume flag when the monitor trap
8943 * flag is pending without being used. Seen early in bios init when
8944 * accessing APIC page in protected mode. */
8945
8946 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8947 return rc;
8948}
8949
8950
8951/**
8952 * Runs the guest code using VT-x.
8953 *
8954 * @returns VBox status code.
8955 * @param pVM Pointer to the VM.
8956 * @param pVCpu Pointer to the VMCPU.
8957 * @param pCtx Pointer to the guest-CPU context.
8958 */
8959VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8960{
8961 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8962 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8963 HMVMX_ASSERT_PREEMPT_SAFE();
8964
8965 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8966
8967 int rc;
8968 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8969 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8970 else
8971 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8972
8973 if (rc == VERR_EM_INTERPRETER)
8974 rc = VINF_EM_RAW_EMULATE_INSTR;
8975 else if (rc == VINF_EM_RESET)
8976 rc = VINF_EM_TRIPLE_FAULT;
8977
8978 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8979 if (RT_FAILURE(rc2))
8980 {
8981 pVCpu->hm.s.u32HMError = rc;
8982 rc = rc2;
8983 }
8984 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8985 return rc;
8986}
8987
8988
8989#ifndef HMVMX_USE_FUNCTION_TABLE
8990DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8991{
8992#ifdef DEBUG_ramshankar
8993# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8994# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8995#endif
8996 int rc;
8997 switch (rcReason)
8998 {
8999 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9000 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9001 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9002 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9003 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9004 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9005 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9006 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9007 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9008 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9009 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9010 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9011 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9012 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9013 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9014 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9015 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9016 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9017 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9018 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9019 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9020 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9021 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9022 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9023 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9024 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9025 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9026 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9027 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9028 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9029 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9030 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9031 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9032 case VMX_EXIT_VMCALL: /* SVVMCS(); */ rc = hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
9033
9034 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
9035 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
9036 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
9037 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
9038 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9039 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
9040 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
9041 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
9042 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
9043
9044 case VMX_EXIT_VMCLEAR:
9045 case VMX_EXIT_VMLAUNCH:
9046 case VMX_EXIT_VMPTRLD:
9047 case VMX_EXIT_VMPTRST:
9048 case VMX_EXIT_VMREAD:
9049 case VMX_EXIT_VMRESUME:
9050 case VMX_EXIT_VMWRITE:
9051 case VMX_EXIT_VMXOFF:
9052 case VMX_EXIT_VMXON:
9053 case VMX_EXIT_INVEPT:
9054 case VMX_EXIT_INVVPID:
9055 case VMX_EXIT_VMFUNC:
9056 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
9057 break;
9058 default:
9059 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
9060 break;
9061 }
9062 return rc;
9063}
9064#endif
9065
9066#ifdef DEBUG
9067/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
9068# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
9069 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
9070
9071# define HMVMX_ASSERT_PREEMPT_CPUID() \
9072 do \
9073 { \
9074 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
9075 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
9076 } while (0)
9077
9078# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9079 do { \
9080 AssertPtr(pVCpu); \
9081 AssertPtr(pMixedCtx); \
9082 AssertPtr(pVmxTransient); \
9083 Assert(pVmxTransient->fVMEntryFailed == false); \
9084 Assert(ASMIntAreEnabled()); \
9085 HMVMX_ASSERT_PREEMPT_SAFE(); \
9086 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
9087 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
9088 HMVMX_ASSERT_PREEMPT_SAFE(); \
9089 if (VMMR0IsLogFlushDisabled(pVCpu)) \
9090 HMVMX_ASSERT_PREEMPT_CPUID(); \
9091 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9092 } while (0)
9093
9094# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
9095 do { \
9096 Log4Func(("\n")); \
9097 } while (0)
9098#else /* Release builds */
9099# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
9100 do { \
9101 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
9102 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
9103 } while (0)
9104# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
9105#endif
9106
9107
9108/**
9109 * Advances the guest RIP after reading it from the VMCS.
9110 *
9111 * @returns VBox status code.
9112 * @param pVCpu Pointer to the VMCPU.
9113 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9114 * out-of-sync. Make sure to update the required fields
9115 * before using them.
9116 * @param pVmxTransient Pointer to the VMX transient structure.
9117 *
9118 * @remarks No-long-jump zone!!!
9119 */
9120DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9121{
9122 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9123 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
9124 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9125 AssertRCReturn(rc, rc);
9126
9127 pMixedCtx->rip += pVmxTransient->cbInstr;
9128 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
9129
9130 /*
9131 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
9132 * pending debug exception field as it takes care of priority of events.
9133 *
9134 * See Intel spec. 32.2.1 "Debug Exceptions".
9135 */
9136 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
9137
9138 return rc;
9139}
9140
9141
9142/**
9143 * Tries to determine what part of the guest-state VT-x has deemed as invalid
9144 * and update error record fields accordingly.
9145 *
9146 * @return VMX_IGS_* return codes.
9147 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
9148 * wrong with the guest state.
9149 *
9150 * @param pVM Pointer to the VM.
9151 * @param pVCpu Pointer to the VMCPU.
9152 * @param pCtx Pointer to the guest-CPU state.
9153 *
9154 * @remarks This function assumes our cache of the VMCS controls
9155 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
9156 */
9157static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9158{
9159#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
9160#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
9161 uError = (err); \
9162 break; \
9163 } else do { } while (0)
9164
9165 int rc;
9166 uint32_t uError = VMX_IGS_ERROR;
9167 uint32_t u32Val;
9168 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
9169
9170 do
9171 {
9172 /*
9173 * CR0.
9174 */
9175 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9176 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
9177 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
9178 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
9179 if (fUnrestrictedGuest)
9180 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
9181
9182 uint32_t u32GuestCR0;
9183 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
9184 AssertRCBreak(rc);
9185 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
9186 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
9187 if ( !fUnrestrictedGuest
9188 && (u32GuestCR0 & X86_CR0_PG)
9189 && !(u32GuestCR0 & X86_CR0_PE))
9190 {
9191 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
9192 }
9193
9194 /*
9195 * CR4.
9196 */
9197 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9198 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9199
9200 uint32_t u32GuestCR4;
9201 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9202 AssertRCBreak(rc);
9203 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9204 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9205
9206 /*
9207 * IA32_DEBUGCTL MSR.
9208 */
9209 uint64_t u64Val;
9210 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9211 AssertRCBreak(rc);
9212 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9213 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9214 {
9215 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9216 }
9217 uint64_t u64DebugCtlMsr = u64Val;
9218
9219#ifdef VBOX_STRICT
9220 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9221 AssertRCBreak(rc);
9222 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9223#endif
9224 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9225
9226 /*
9227 * RIP and RFLAGS.
9228 */
9229 uint32_t u32Eflags;
9230#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9231 if (HMVMX_IS_64BIT_HOST_MODE())
9232 {
9233 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9234 AssertRCBreak(rc);
9235 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9236 if ( !fLongModeGuest
9237 || !pCtx->cs.Attr.n.u1Long)
9238 {
9239 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9240 }
9241 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9242 * must be identical if the "IA-32e mode guest" VM-entry
9243 * control is 1 and CS.L is 1. No check applies if the
9244 * CPU supports 64 linear-address bits. */
9245
9246 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9247 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9248 AssertRCBreak(rc);
9249 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9250 VMX_IGS_RFLAGS_RESERVED);
9251 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9252 u32Eflags = u64Val;
9253 }
9254 else
9255#endif
9256 {
9257 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9258 AssertRCBreak(rc);
9259 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9260 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9261 }
9262
9263 if ( fLongModeGuest
9264 || ( fUnrestrictedGuest
9265 && !(u32GuestCR0 & X86_CR0_PE)))
9266 {
9267 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9268 }
9269
9270 uint32_t u32EntryInfo;
9271 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9272 AssertRCBreak(rc);
9273 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9274 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9275 {
9276 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9277 }
9278
9279 /*
9280 * 64-bit checks.
9281 */
9282#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9283 if (HMVMX_IS_64BIT_HOST_MODE())
9284 {
9285 if ( fLongModeGuest
9286 && !fUnrestrictedGuest)
9287 {
9288 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9289 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9290 }
9291
9292 if ( !fLongModeGuest
9293 && (u32GuestCR4 & X86_CR4_PCIDE))
9294 {
9295 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9296 }
9297
9298 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9299 * 51:32 beyond the processor's physical-address width are 0. */
9300
9301 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9302 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9303 {
9304 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9305 }
9306
9307 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9308 AssertRCBreak(rc);
9309 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9310
9311 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9312 AssertRCBreak(rc);
9313 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9314 }
9315#endif
9316
9317 /*
9318 * PERF_GLOBAL MSR.
9319 */
9320 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9321 {
9322 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9323 AssertRCBreak(rc);
9324 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9325 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9326 }
9327
9328 /*
9329 * PAT MSR.
9330 */
9331 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9332 {
9333 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9334 AssertRCBreak(rc);
9335 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9336 for (unsigned i = 0; i < 8; i++)
9337 {
9338 uint8_t u8Val = (u64Val & 0x7);
9339 if ( u8Val != 0 /* UC */
9340 || u8Val != 1 /* WC */
9341 || u8Val != 4 /* WT */
9342 || u8Val != 5 /* WP */
9343 || u8Val != 6 /* WB */
9344 || u8Val != 7 /* UC- */)
9345 {
9346 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9347 }
9348 u64Val >>= 3;
9349 }
9350 }
9351
9352 /*
9353 * EFER MSR.
9354 */
9355 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9356 {
9357 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9358 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9359 AssertRCBreak(rc);
9360 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9361 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9362 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9363 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9364 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9365 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9366 VMX_IGS_EFER_LMA_PG_MISMATCH);
9367 }
9368
9369 /*
9370 * Segment registers.
9371 */
9372 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9373 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9374 if (!(u32Eflags & X86_EFL_VM))
9375 {
9376 /* CS */
9377 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9378 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9379 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9380 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9381 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9382 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9383 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9384 /* CS cannot be loaded with NULL in protected mode. */
9385 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9386 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9387 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9388 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9389 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9390 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9391 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9392 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9393 else
9394 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9395
9396 /* SS */
9397 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9398 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9399 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9400 if ( !(pCtx->cr0 & X86_CR0_PE)
9401 || pCtx->cs.Attr.n.u4Type == 3)
9402 {
9403 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9404 }
9405 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9406 {
9407 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9408 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9409 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9410 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9411 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9412 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9413 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9414 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9415 }
9416
9417 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9418 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9419 {
9420 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9421 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9422 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9423 || pCtx->ds.Attr.n.u4Type > 11
9424 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9425 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9426 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9427 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9428 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9429 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9430 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9431 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9432 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9433 }
9434 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9435 {
9436 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9437 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9438 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9439 || pCtx->es.Attr.n.u4Type > 11
9440 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9441 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9442 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9443 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9444 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9445 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9446 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9447 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9448 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9449 }
9450 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9451 {
9452 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9453 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9454 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9455 || pCtx->fs.Attr.n.u4Type > 11
9456 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9457 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9458 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9459 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9460 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9461 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9462 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9463 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9464 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9465 }
9466 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9467 {
9468 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9469 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9470 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9471 || pCtx->gs.Attr.n.u4Type > 11
9472 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9473 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9474 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9475 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9476 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9477 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9478 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9479 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9480 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9481 }
9482 /* 64-bit capable CPUs. */
9483#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9484 if (HMVMX_IS_64BIT_HOST_MODE())
9485 {
9486 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9487 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9488 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9489 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9490 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9491 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9492 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9493 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9494 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9495 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9496 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9497 }
9498#endif
9499 }
9500 else
9501 {
9502 /* V86 mode checks. */
9503 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9504 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9505 {
9506 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9507 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9508 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9509 }
9510 else
9511 {
9512 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9513 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9514 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9515 }
9516
9517 /* CS */
9518 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9519 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9520 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9521 /* SS */
9522 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9523 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9524 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9525 /* DS */
9526 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9527 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9528 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9529 /* ES */
9530 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9531 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9532 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9533 /* FS */
9534 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9535 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9536 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9537 /* GS */
9538 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9539 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9540 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9541 /* 64-bit capable CPUs. */
9542#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9543 if (HMVMX_IS_64BIT_HOST_MODE())
9544 {
9545 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9546 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9547 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9548 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9549 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9550 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9551 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9552 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9553 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9554 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9555 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9556 }
9557#endif
9558 }
9559
9560 /*
9561 * TR.
9562 */
9563 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9564 /* 64-bit capable CPUs. */
9565#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9566 if (HMVMX_IS_64BIT_HOST_MODE())
9567 {
9568 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9569 }
9570#endif
9571 if (fLongModeGuest)
9572 {
9573 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9574 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9575 }
9576 else
9577 {
9578 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9579 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9580 VMX_IGS_TR_ATTR_TYPE_INVALID);
9581 }
9582 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9583 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9584 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9585 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9586 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9587 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9588 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9589 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9590
9591 /*
9592 * GDTR and IDTR.
9593 */
9594#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9595 if (HMVMX_IS_64BIT_HOST_MODE())
9596 {
9597 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9598 AssertRCBreak(rc);
9599 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9600
9601 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9602 AssertRCBreak(rc);
9603 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9604 }
9605#endif
9606
9607 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9608 AssertRCBreak(rc);
9609 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9610
9611 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9612 AssertRCBreak(rc);
9613 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9614
9615 /*
9616 * Guest Non-Register State.
9617 */
9618 /* Activity State. */
9619 uint32_t u32ActivityState;
9620 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9621 AssertRCBreak(rc);
9622 HMVMX_CHECK_BREAK( !u32ActivityState
9623 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9624 VMX_IGS_ACTIVITY_STATE_INVALID);
9625 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9626 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9627 uint32_t u32IntrState;
9628 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9629 AssertRCBreak(rc);
9630 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9631 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9632 {
9633 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9634 }
9635
9636 /** @todo Activity state and injecting interrupts. Left as a todo since we
9637 * currently don't use activity states but ACTIVE. */
9638
9639 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9640 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9641
9642 /* Guest interruptibility-state. */
9643 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9644 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9645 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9646 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9647 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9648 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9649 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9650 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9651 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9652 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9653 {
9654 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9655 {
9656 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9657 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9658 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9659 }
9660 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9661 {
9662 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9663 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9664 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9665 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9666 }
9667 }
9668 /** @todo Assumes the processor is not in SMM. */
9669 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9670 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9671 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9672 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9673 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9674 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9675 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9676 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9677 {
9678 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9679 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9680 }
9681
9682 /* Pending debug exceptions. */
9683 if (HMVMX_IS_64BIT_HOST_MODE())
9684 {
9685 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9686 AssertRCBreak(rc);
9687 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9688 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9689 u32Val = u64Val; /* For pending debug exceptions checks below. */
9690 }
9691 else
9692 {
9693 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9694 AssertRCBreak(rc);
9695 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9696 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9697 }
9698
9699 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9700 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9701 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9702 {
9703 if ( (u32Eflags & X86_EFL_TF)
9704 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9705 {
9706 /* Bit 14 is PendingDebug.BS. */
9707 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9708 }
9709 if ( !(u32Eflags & X86_EFL_TF)
9710 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9711 {
9712 /* Bit 14 is PendingDebug.BS. */
9713 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9714 }
9715 }
9716
9717 /* VMCS link pointer. */
9718 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9719 AssertRCBreak(rc);
9720 if (u64Val != UINT64_C(0xffffffffffffffff))
9721 {
9722 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9723 /** @todo Bits beyond the processor's physical-address width MBZ. */
9724 /** @todo 32-bit located in memory referenced by value of this field (as a
9725 * physical address) must contain the processor's VMCS revision ID. */
9726 /** @todo SMM checks. */
9727 }
9728
9729 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9730 * not using Nested Paging? */
9731 if ( pVM->hm.s.fNestedPaging
9732 && !fLongModeGuest
9733 && CPUMIsGuestInPAEModeEx(pCtx))
9734 {
9735 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9736 AssertRCBreak(rc);
9737 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9738
9739 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9740 AssertRCBreak(rc);
9741 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9742
9743 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9744 AssertRCBreak(rc);
9745 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9746
9747 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9748 AssertRCBreak(rc);
9749 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9750 }
9751
9752 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9753 if (uError == VMX_IGS_ERROR)
9754 uError = VMX_IGS_REASON_NOT_FOUND;
9755 } while (0);
9756
9757 pVCpu->hm.s.u32HMError = uError;
9758 return uError;
9759
9760#undef HMVMX_ERROR_BREAK
9761#undef HMVMX_CHECK_BREAK
9762}
9763
9764/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9765/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9766/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9767
9768/** @name VM-exit handlers.
9769 * @{
9770 */
9771
9772/**
9773 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9774 */
9775HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9776{
9777 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9778 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9779 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9780 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9781 return VINF_SUCCESS;
9782 return VINF_EM_RAW_INTERRUPT;
9783}
9784
9785
9786/**
9787 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9788 */
9789HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9790{
9791 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9792 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9793
9794 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9795 AssertRCReturn(rc, rc);
9796
9797 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9798 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9799 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9800 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9801
9802 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9803 {
9804 /*
9805 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9806 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9807 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9808 *
9809 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9810 */
9811 VMXDispatchHostNmi();
9812 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9813 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9814 return VINF_SUCCESS;
9815 }
9816
9817 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9818 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9819 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9820 {
9821 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9822 return VINF_SUCCESS;
9823 }
9824 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9825 {
9826 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9827 return rc;
9828 }
9829
9830 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9831 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9832 switch (uIntType)
9833 {
9834 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9835 Assert(uVector == X86_XCPT_DB);
9836 /* no break */
9837 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9838 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9839 /* no break */
9840 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9841 {
9842 switch (uVector)
9843 {
9844 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9845 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9846 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9847 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9848 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9849 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9850#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9851 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9852 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9853 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9854 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9855 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9856 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9857 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9858 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9859 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9860 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9861 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9862 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9863#endif
9864 default:
9865 {
9866 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9867 AssertRCReturn(rc, rc);
9868
9869 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9870 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9871 {
9872 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9873 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9874 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9875
9876 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9877 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9878 AssertRCReturn(rc, rc);
9879 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9880 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9881 0 /* GCPtrFaultAddress */);
9882 AssertRCReturn(rc, rc);
9883 }
9884 else
9885 {
9886 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9887 pVCpu->hm.s.u32HMError = uVector;
9888 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9889 }
9890 break;
9891 }
9892 }
9893 break;
9894 }
9895
9896 default:
9897 {
9898 pVCpu->hm.s.u32HMError = uExitIntInfo;
9899 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9900 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9901 break;
9902 }
9903 }
9904 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9905 return rc;
9906}
9907
9908
9909/**
9910 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9911 */
9912HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9913{
9914 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9915
9916 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9917 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9918
9919 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9920 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9921 return VINF_SUCCESS;
9922}
9923
9924
9925/**
9926 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9927 */
9928HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9929{
9930 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9931 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
9932 {
9933 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9934 HMVMX_RETURN_UNEXPECTED_EXIT();
9935 }
9936
9937 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
9938
9939 /*
9940 * Clear block-by-STI if it's active. The force-flag couldn't have been set by block-by-Mov SS in
9941 * hmR0VmxSaveGuestIntrState() when this VM-exit happens as Intel CPUs are consistent with
9942 * block-by-Mov SS and NMIs. See @bugref{7445}.
9943 */
9944 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
9945 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
9946
9947 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
9948 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
9949
9950 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
9951 return VINF_SUCCESS;
9952}
9953
9954
9955/**
9956 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9957 */
9958HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9959{
9960 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9962 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9963}
9964
9965
9966/**
9967 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9968 */
9969HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9970{
9971 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9973 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9974}
9975
9976
9977/**
9978 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9979 */
9980HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9981{
9982 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9983 PVM pVM = pVCpu->CTX_SUFF(pVM);
9984 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9985 if (RT_LIKELY(rc == VINF_SUCCESS))
9986 {
9987 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9988 Assert(pVmxTransient->cbInstr == 2);
9989 }
9990 else
9991 {
9992 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9993 rc = VERR_EM_INTERPRETER;
9994 }
9995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9996 return rc;
9997}
9998
9999
10000/**
10001 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
10002 */
10003HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10004{
10005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10006 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
10007 AssertRCReturn(rc, rc);
10008
10009 if (pMixedCtx->cr4 & X86_CR4_SMXE)
10010 return VINF_EM_RAW_EMULATE_INSTR;
10011
10012 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
10013 HMVMX_RETURN_UNEXPECTED_EXIT();
10014}
10015
10016
10017/**
10018 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
10019 */
10020HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10021{
10022 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10023 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10024 AssertRCReturn(rc, rc);
10025
10026 PVM pVM = pVCpu->CTX_SUFF(pVM);
10027 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10028 if (RT_LIKELY(rc == VINF_SUCCESS))
10029 {
10030 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10031 Assert(pVmxTransient->cbInstr == 2);
10032 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10033 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10034 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10035 }
10036 else
10037 {
10038 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
10039 rc = VERR_EM_INTERPRETER;
10040 }
10041 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10042 return rc;
10043}
10044
10045
10046/**
10047 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
10048 */
10049HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10050{
10051 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10052 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10053 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
10054 AssertRCReturn(rc, rc);
10055
10056 PVM pVM = pVCpu->CTX_SUFF(pVM);
10057 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
10058 if (RT_LIKELY(rc == VINF_SUCCESS))
10059 {
10060 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10061 Assert(pVmxTransient->cbInstr == 3);
10062 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
10063 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
10064 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10065 }
10066 else
10067 {
10068 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
10069 rc = VERR_EM_INTERPRETER;
10070 }
10071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
10072 return rc;
10073}
10074
10075
10076/**
10077 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
10078 */
10079HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10080{
10081 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10082 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
10083 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
10084 AssertRCReturn(rc, rc);
10085
10086 PVM pVM = pVCpu->CTX_SUFF(pVM);
10087 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10088 if (RT_LIKELY(rc == VINF_SUCCESS))
10089 {
10090 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10091 Assert(pVmxTransient->cbInstr == 2);
10092 }
10093 else
10094 {
10095 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
10096 rc = VERR_EM_INTERPRETER;
10097 }
10098 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
10099 return rc;
10100}
10101
10102
10103/**
10104 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
10105 */
10106HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10107{
10108 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10109
10110 int rc = VERR_NOT_SUPPORTED;
10111 if (GIMAreHypercallsEnabled(pVCpu))
10112 {
10113 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10114 AssertRCReturn(rc, rc);
10115
10116 rc = GIMHypercall(pVCpu, pMixedCtx);
10117 }
10118 if (rc != VINF_SUCCESS)
10119 {
10120 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10121 rc = VINF_SUCCESS;
10122 }
10123
10124 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
10125 return rc;
10126}
10127
10128
10129/**
10130 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
10131 */
10132HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10133{
10134 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10135 PVM pVM = pVCpu->CTX_SUFF(pVM);
10136 Assert(!pVM->hm.s.fNestedPaging);
10137
10138 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10139 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10140 AssertRCReturn(rc, rc);
10141
10142 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
10143 rc = VBOXSTRICTRC_VAL(rc2);
10144 if (RT_LIKELY(rc == VINF_SUCCESS))
10145 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10146 else
10147 {
10148 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
10149 pVmxTransient->uExitQualification, rc));
10150 }
10151 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
10152 return rc;
10153}
10154
10155
10156/**
10157 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
10158 */
10159HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10160{
10161 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10162 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10163 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10164 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10165 AssertRCReturn(rc, rc);
10166
10167 PVM pVM = pVCpu->CTX_SUFF(pVM);
10168 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10169 if (RT_LIKELY(rc == VINF_SUCCESS))
10170 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10171 else
10172 {
10173 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
10174 rc = VERR_EM_INTERPRETER;
10175 }
10176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
10177 return rc;
10178}
10179
10180
10181/**
10182 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
10183 */
10184HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10185{
10186 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10187 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10188 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10189 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10190 AssertRCReturn(rc, rc);
10191
10192 PVM pVM = pVCpu->CTX_SUFF(pVM);
10193 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10194 rc = VBOXSTRICTRC_VAL(rc2);
10195 if (RT_LIKELY( rc == VINF_SUCCESS
10196 || rc == VINF_EM_HALT))
10197 {
10198 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10199 AssertRCReturn(rc3, rc3);
10200
10201 if ( rc == VINF_EM_HALT
10202 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
10203 {
10204 rc = VINF_SUCCESS;
10205 }
10206 }
10207 else
10208 {
10209 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
10210 rc = VERR_EM_INTERPRETER;
10211 }
10212 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
10213 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
10214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
10215 return rc;
10216}
10217
10218
10219/**
10220 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
10221 */
10222HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10223{
10224 /*
10225 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
10226 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
10227 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
10228 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
10229 */
10230 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10231 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10232 HMVMX_RETURN_UNEXPECTED_EXIT();
10233}
10234
10235
10236/**
10237 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
10238 */
10239HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10240{
10241 /*
10242 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10243 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10244 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10245 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10246 */
10247 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10248 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10249 HMVMX_RETURN_UNEXPECTED_EXIT();
10250}
10251
10252
10253/**
10254 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10255 */
10256HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10257{
10258 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10259 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10260 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10261 HMVMX_RETURN_UNEXPECTED_EXIT();
10262}
10263
10264
10265/**
10266 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10267 */
10268HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10269{
10270 /*
10271 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10272 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10273 * See Intel spec. 25.3 "Other Causes of VM-exits".
10274 */
10275 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10276 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10277 HMVMX_RETURN_UNEXPECTED_EXIT();
10278}
10279
10280
10281/**
10282 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10283 * VM-exit.
10284 */
10285HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10286{
10287 /*
10288 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10289 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10290 *
10291 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10292 * See Intel spec. "23.8 Restrictions on VMX operation".
10293 */
10294 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10295 return VINF_SUCCESS;
10296}
10297
10298
10299/**
10300 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10301 * VM-exit.
10302 */
10303HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10304{
10305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10306 return VINF_EM_RESET;
10307}
10308
10309
10310/**
10311 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10312 */
10313HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10314{
10315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10316 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10317 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10318 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10319 AssertRCReturn(rc, rc);
10320
10321 pMixedCtx->rip++;
10322 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10323 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10324 rc = VINF_SUCCESS;
10325 else
10326 rc = VINF_EM_HALT;
10327
10328 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10329 return rc;
10330}
10331
10332
10333/**
10334 * VM-exit handler for instructions that result in a #UD exception delivered to
10335 * the guest.
10336 */
10337HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10338{
10339 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10340 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10341 return VINF_SUCCESS;
10342}
10343
10344
10345/**
10346 * VM-exit handler for expiry of the VMX preemption timer.
10347 */
10348HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10349{
10350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10351
10352 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10353 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10354
10355 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10356 PVM pVM = pVCpu->CTX_SUFF(pVM);
10357 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10358 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10359 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10360}
10361
10362
10363/**
10364 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10365 */
10366HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10367{
10368 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10369
10370 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10371 /** @todo check if XSETBV is supported by the recompiler. */
10372 return VERR_EM_INTERPRETER;
10373}
10374
10375
10376/**
10377 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10378 */
10379HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10380{
10381 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10382
10383 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10384 /** @todo implement EMInterpretInvpcid() */
10385 return VERR_EM_INTERPRETER;
10386}
10387
10388
10389/**
10390 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10391 * Error VM-exit.
10392 */
10393HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10394{
10395 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10396 AssertRCReturn(rc, rc);
10397
10398 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10399 AssertRCReturn(rc, rc);
10400
10401 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10402 NOREF(uInvalidReason);
10403
10404#ifdef VBOX_STRICT
10405 uint32_t uIntrState;
10406 HMVMXHCUINTREG uHCReg;
10407 uint64_t u64Val;
10408 uint32_t u32Val;
10409
10410 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10411 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10412 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10413 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10414 AssertRCReturn(rc, rc);
10415
10416 Log4(("uInvalidReason %u\n", uInvalidReason));
10417 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10418 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10419 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10420 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10421
10422 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10423 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10424 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10425 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10426 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10427 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10428 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10429 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10430 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10431 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10432 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10433 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10434#else
10435 NOREF(pVmxTransient);
10436#endif
10437
10438 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10439 return VERR_VMX_INVALID_GUEST_STATE;
10440}
10441
10442
10443/**
10444 * VM-exit handler for VM-entry failure due to an MSR-load
10445 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10446 */
10447HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10448{
10449 NOREF(pVmxTransient);
10450 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10451 HMVMX_RETURN_UNEXPECTED_EXIT();
10452}
10453
10454
10455/**
10456 * VM-exit handler for VM-entry failure due to a machine-check event
10457 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10458 */
10459HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10460{
10461 NOREF(pVmxTransient);
10462 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10463 HMVMX_RETURN_UNEXPECTED_EXIT();
10464}
10465
10466
10467/**
10468 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10469 * theory.
10470 */
10471HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10472{
10473 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10474 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10475 return VERR_VMX_UNDEFINED_EXIT_CODE;
10476}
10477
10478
10479/**
10480 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10481 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10482 * Conditional VM-exit.
10483 */
10484HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10485{
10486 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10487
10488 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10489 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10490 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10491 return VERR_EM_INTERPRETER;
10492 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10493 HMVMX_RETURN_UNEXPECTED_EXIT();
10494}
10495
10496
10497/**
10498 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10499 */
10500HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10501{
10502 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10503
10504 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10505 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10506 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10507 return VERR_EM_INTERPRETER;
10508 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10509 HMVMX_RETURN_UNEXPECTED_EXIT();
10510}
10511
10512
10513/**
10514 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10515 */
10516HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10517{
10518 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10519
10520 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10521 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10522 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10523 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10524 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10525 {
10526 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10527 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10528 }
10529 AssertRCReturn(rc, rc);
10530 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10531
10532#ifdef VBOX_STRICT
10533 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10534 {
10535 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
10536 && pMixedCtx->ecx != MSR_K6_EFER)
10537 {
10538 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10539 HMVMX_RETURN_UNEXPECTED_EXIT();
10540 }
10541# if HC_ARCH_BITS == 64
10542 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10543 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10544 {
10545 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10546 HMVMX_RETURN_UNEXPECTED_EXIT();
10547 }
10548# endif
10549 }
10550#endif
10551
10552 PVM pVM = pVCpu->CTX_SUFF(pVM);
10553 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10554 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10555 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10556 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10557 if (RT_LIKELY(rc == VINF_SUCCESS))
10558 {
10559 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10560 Assert(pVmxTransient->cbInstr == 2);
10561 }
10562 return rc;
10563}
10564
10565
10566/**
10567 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10568 */
10569HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10570{
10571 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10572 PVM pVM = pVCpu->CTX_SUFF(pVM);
10573 int rc = VINF_SUCCESS;
10574
10575 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10576 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10577 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10578 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10579 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10580 {
10581 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10582 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10583 }
10584 AssertRCReturn(rc, rc);
10585 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10586
10587 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10588 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10589 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10590
10591 if (RT_LIKELY(rc == VINF_SUCCESS))
10592 {
10593 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10594
10595 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10596 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10597 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10598 {
10599 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10600 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10601 EMInterpretWrmsr() changes it. */
10602 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10603 }
10604 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10605 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10606 else if (pMixedCtx->ecx == MSR_K6_EFER)
10607 {
10608 /*
10609 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10610 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10611 * the other bits as well, SCE and NXE. See @bugref{7368}.
10612 */
10613 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10614 }
10615
10616 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10617 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10618 {
10619 switch (pMixedCtx->ecx)
10620 {
10621 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10622 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10623 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10624 case MSR_K8_FS_BASE: /* no break */
10625 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10626 case MSR_K6_EFER: /* already handled above */ break;
10627 default:
10628 {
10629 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10630 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10631#if HC_ARCH_BITS == 64
10632 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10633 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10634#endif
10635 break;
10636 }
10637 }
10638 }
10639#ifdef VBOX_STRICT
10640 else
10641 {
10642 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10643 switch (pMixedCtx->ecx)
10644 {
10645 case MSR_IA32_SYSENTER_CS:
10646 case MSR_IA32_SYSENTER_EIP:
10647 case MSR_IA32_SYSENTER_ESP:
10648 case MSR_K8_FS_BASE:
10649 case MSR_K8_GS_BASE:
10650 {
10651 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10652 HMVMX_RETURN_UNEXPECTED_EXIT();
10653 }
10654
10655 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10656 default:
10657 {
10658 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10659 {
10660 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
10661 if (pMixedCtx->ecx != MSR_K6_EFER)
10662 {
10663 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10664 pMixedCtx->ecx));
10665 HMVMX_RETURN_UNEXPECTED_EXIT();
10666 }
10667 }
10668
10669#if HC_ARCH_BITS == 64
10670 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10671 {
10672 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10673 HMVMX_RETURN_UNEXPECTED_EXIT();
10674 }
10675#endif
10676 break;
10677 }
10678 }
10679 }
10680#endif /* VBOX_STRICT */
10681 }
10682 return rc;
10683}
10684
10685
10686/**
10687 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10688 */
10689HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10690{
10691 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10692
10693 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10695 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10696 return VERR_EM_INTERPRETER;
10697 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10698 HMVMX_RETURN_UNEXPECTED_EXIT();
10699}
10700
10701
10702/**
10703 * VM-exit handler for when the TPR value is lowered below the specified
10704 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10705 */
10706HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10707{
10708 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10709 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10710
10711 /*
10712 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10713 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10714 * resume guest execution.
10715 */
10716 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10718 return VINF_SUCCESS;
10719}
10720
10721
10722/**
10723 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10724 * VM-exit.
10725 *
10726 * @retval VINF_SUCCESS when guest execution can continue.
10727 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10728 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10729 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10730 * recompiler.
10731 */
10732HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10733{
10734 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10735 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10736 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10737 AssertRCReturn(rc, rc);
10738
10739 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
10740 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10741 PVM pVM = pVCpu->CTX_SUFF(pVM);
10742 switch (uAccessType)
10743 {
10744 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10745 {
10746#if 0
10747 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10748 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10749#else
10750 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10751 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10752 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10753#endif
10754 AssertRCReturn(rc, rc);
10755
10756 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10757 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10758 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10759 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10760
10761 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10762 {
10763 case 0: /* CR0 */
10764 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10765 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10766 break;
10767 case 2: /* CR2 */
10768 /* Nothing to do here, CR2 it's not part of the VMCS. */
10769 break;
10770 case 3: /* CR3 */
10771 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10772 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10773 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10774 break;
10775 case 4: /* CR4 */
10776 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10777 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10778 break;
10779 case 8: /* CR8 */
10780 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10781 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10782 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10783 break;
10784 default:
10785 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10786 break;
10787 }
10788
10789 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10790 break;
10791 }
10792
10793 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10794 {
10795 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10796 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10797 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10798 AssertRCReturn(rc, rc);
10799 Assert( !pVM->hm.s.fNestedPaging
10800 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10801 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10802
10803 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10804 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10805 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10806
10807 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10808 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10809 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10810 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10812 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10813 break;
10814 }
10815
10816 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10817 {
10818 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10819 AssertRCReturn(rc, rc);
10820 rc = EMInterpretCLTS(pVM, pVCpu);
10821 AssertRCReturn(rc, rc);
10822 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10824 Log4(("CRX CLTS write rc=%d\n", rc));
10825 break;
10826 }
10827
10828 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10829 {
10830 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10831 AssertRCReturn(rc, rc);
10832 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10833 if (RT_LIKELY(rc == VINF_SUCCESS))
10834 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10835 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10836 Log4(("CRX LMSW write rc=%d\n", rc));
10837 break;
10838 }
10839
10840 default:
10841 {
10842 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10843 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10844 }
10845 }
10846
10847 /* Validate possible error codes. */
10848 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10849 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10850 if (RT_SUCCESS(rc))
10851 {
10852 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10853 AssertRCReturn(rc2, rc2);
10854 }
10855
10856 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10857 return rc;
10858}
10859
10860
10861/**
10862 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10863 * VM-exit.
10864 */
10865HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10866{
10867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10868 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10869
10870 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10871 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10872 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10873 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10874 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10875 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10876 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10877 AssertRCReturn(rc2, rc2);
10878
10879 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10880 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10881 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10882 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10883 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10884 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10885 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10886 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
10887
10888 /* I/O operation lookup arrays. */
10889 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10890 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10891
10892 VBOXSTRICTRC rcStrict;
10893 uint32_t const cbValue = s_aIOSizes[uIOWidth];
10894 uint32_t const cbInstr = pVmxTransient->cbInstr;
10895 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10896 PVM pVM = pVCpu->CTX_SUFF(pVM);
10897 if (fIOString)
10898 {
10899#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10900 /*
10901 * INS/OUTS - I/O String instruction.
10902 *
10903 * Use instruction-information if available, otherwise fall back on
10904 * interpreting the instruction.
10905 */
10906 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10907 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
10908 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10909 {
10910 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10911 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10912 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10913 AssertRCReturn(rc2, rc2);
10914 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
10915 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10916 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10917 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10918 if (fIOWrite)
10919 {
10920 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10921 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10922 }
10923 else
10924 {
10925 /*
10926 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10927 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10928 * See Intel Instruction spec. for "INS".
10929 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10930 */
10931 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10932 }
10933 }
10934 else
10935 {
10936 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10937 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10938 AssertRCReturn(rc2, rc2);
10939 rcStrict = IEMExecOne(pVCpu);
10940 }
10941 /** @todo IEM needs to be setting these flags somehow. */
10942 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10943 fUpdateRipAlready = true;
10944#else
10945 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10946 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10947 if (RT_SUCCESS(rcStrict))
10948 {
10949 if (fIOWrite)
10950 {
10951 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10952 (DISCPUMODE)pDis->uAddrMode, cbValue);
10953 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10954 }
10955 else
10956 {
10957 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10958 (DISCPUMODE)pDis->uAddrMode, cbValue);
10959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10960 }
10961 }
10962 else
10963 {
10964 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10965 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10966 }
10967#endif
10968 }
10969 else
10970 {
10971 /*
10972 * IN/OUT - I/O instruction.
10973 */
10974 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10975 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
10976 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10977 if (fIOWrite)
10978 {
10979 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10980 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10981 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10983 }
10984 else
10985 {
10986 uint32_t u32Result = 0;
10987 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10988 if (IOM_SUCCESS(rcStrict))
10989 {
10990 /* Save result of I/O IN instr. in AL/AX/EAX. */
10991 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10992 }
10993 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10994 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10996 }
10997 }
10998
10999 if (IOM_SUCCESS(rcStrict))
11000 {
11001 if (!fUpdateRipAlready)
11002 {
11003 pMixedCtx->rip += cbInstr;
11004 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11005 }
11006
11007 /*
11008 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
11009 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
11010 */
11011 if (fIOString)
11012 {
11013 /** @todo Single-step for INS/OUTS with REP prefix? */
11014 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
11015 }
11016 else if (fStepping)
11017 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11018
11019 /*
11020 * If any I/O breakpoints are armed, we need to check if one triggered
11021 * and take appropriate action.
11022 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
11023 */
11024 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11025 AssertRCReturn(rc2, rc2);
11026
11027 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
11028 * execution engines about whether hyper BPs and such are pending. */
11029 uint32_t const uDr7 = pMixedCtx->dr[7];
11030 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
11031 && X86_DR7_ANY_RW_IO(uDr7)
11032 && (pMixedCtx->cr4 & X86_CR4_DE))
11033 || DBGFBpIsHwIoArmed(pVM)))
11034 {
11035 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
11036
11037 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
11038 VMMRZCallRing3Disable(pVCpu);
11039 HM_DISABLE_PREEMPT_IF_NEEDED();
11040
11041 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
11042
11043 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
11044 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
11045 {
11046 /* Raise #DB. */
11047 if (fIsGuestDbgActive)
11048 ASMSetDR6(pMixedCtx->dr[6]);
11049 if (pMixedCtx->dr[7] != uDr7)
11050 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11051
11052 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
11053 }
11054 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
11055 else if ( rcStrict2 != VINF_SUCCESS
11056 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
11057 rcStrict = rcStrict2;
11058
11059 HM_RESTORE_PREEMPT_IF_NEEDED();
11060 VMMRZCallRing3Enable(pVCpu);
11061 }
11062 }
11063
11064#ifdef DEBUG
11065 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
11066 Assert(!fIOWrite);
11067 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
11068 Assert(fIOWrite);
11069 else
11070 {
11071 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
11072 * statuses, that the VMM device and some others may return. See
11073 * IOM_SUCCESS() for guidance. */
11074 AssertMsg( RT_FAILURE(rcStrict)
11075 || rcStrict == VINF_SUCCESS
11076 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
11077 || rcStrict == VINF_EM_DBG_BREAKPOINT
11078 || rcStrict == VINF_EM_RAW_GUEST_TRAP
11079 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11080 }
11081#endif
11082
11083 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
11084 return VBOXSTRICTRC_TODO(rcStrict);
11085}
11086
11087
11088/**
11089 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
11090 * VM-exit.
11091 */
11092HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11093{
11094 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11095
11096 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
11097 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11098 AssertRCReturn(rc, rc);
11099 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
11100 {
11101 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
11102 AssertRCReturn(rc, rc);
11103 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
11104 {
11105 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
11106
11107 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
11108 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
11109
11110 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
11111 Assert(!pVCpu->hm.s.Event.fPending);
11112 pVCpu->hm.s.Event.fPending = true;
11113 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
11114 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
11115 AssertRCReturn(rc, rc);
11116 if (fErrorCodeValid)
11117 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
11118 else
11119 pVCpu->hm.s.Event.u32ErrCode = 0;
11120 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
11121 && uVector == X86_XCPT_PF)
11122 {
11123 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
11124 }
11125
11126 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
11127 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11128 return VINF_EM_RAW_INJECT_TRPM_EVENT;
11129 }
11130 }
11131
11132 /** @todo Emulate task switch someday, currently just going back to ring-3 for
11133 * emulation. */
11134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
11135 return VERR_EM_INTERPRETER;
11136}
11137
11138
11139/**
11140 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
11141 */
11142HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11143{
11144 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11145 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
11146 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
11147 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11148 AssertRCReturn(rc, rc);
11149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
11150 return VINF_EM_DBG_STEPPED;
11151}
11152
11153
11154/**
11155 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
11156 */
11157HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11158{
11159 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11160
11161 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11162 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11163 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11164 return VINF_SUCCESS;
11165 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11166 return rc;
11167
11168#if 0
11169 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
11170 * just sync the whole thing. */
11171 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11172#else
11173 /* Aggressive state sync. for now. */
11174 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11175 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11176 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11177#endif
11178 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11179 AssertRCReturn(rc, rc);
11180
11181 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
11182 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
11183 switch (uAccessType)
11184 {
11185 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
11186 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
11187 {
11188 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
11189 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
11190 {
11191 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
11192 }
11193
11194 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
11195 GCPhys &= PAGE_BASE_GC_MASK;
11196 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
11197 PVM pVM = pVCpu->CTX_SUFF(pVM);
11198 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
11199 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
11200
11201 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
11202 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
11203 CPUMCTX2CORE(pMixedCtx), GCPhys);
11204 rc = VBOXSTRICTRC_VAL(rc2);
11205 Log4(("ApicAccess rc=%d\n", rc));
11206 if ( rc == VINF_SUCCESS
11207 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11208 || rc == VERR_PAGE_NOT_PRESENT)
11209 {
11210 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11211 | HM_CHANGED_GUEST_RSP
11212 | HM_CHANGED_GUEST_RFLAGS
11213 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11214 rc = VINF_SUCCESS;
11215 }
11216 break;
11217 }
11218
11219 default:
11220 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
11221 rc = VINF_EM_RAW_EMULATE_INSTR;
11222 break;
11223 }
11224
11225 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
11226 return rc;
11227}
11228
11229
11230/**
11231 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
11232 * VM-exit.
11233 */
11234HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11235{
11236 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11237
11238 /* We should -not- get this VM-exit if the guest's debug registers were active. */
11239 if (pVmxTransient->fWasGuestDebugStateActive)
11240 {
11241 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11242 HMVMX_RETURN_UNEXPECTED_EXIT();
11243 }
11244
11245 int rc = VERR_INTERNAL_ERROR_5;
11246 if ( !DBGFIsStepping(pVCpu)
11247 && !pVCpu->hm.s.fSingleInstruction
11248 && !pVmxTransient->fWasHyperDebugStateActive)
11249 {
11250 /* Don't intercept MOV DRx and #DB any more. */
11251 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11252 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11253 AssertRCReturn(rc, rc);
11254
11255 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11256 {
11257#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11258 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11259 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11260 AssertRCReturn(rc, rc);
11261#endif
11262 }
11263
11264 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11265 VMMRZCallRing3Disable(pVCpu);
11266 HM_DISABLE_PREEMPT_IF_NEEDED();
11267
11268 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11269 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11270 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11271
11272 HM_RESTORE_PREEMPT_IF_NEEDED();
11273 VMMRZCallRing3Enable(pVCpu);
11274
11275#ifdef VBOX_WITH_STATISTICS
11276 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11277 AssertRCReturn(rc, rc);
11278 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11280 else
11281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11282#endif
11283 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11284 return VINF_SUCCESS;
11285 }
11286
11287 /*
11288 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11289 * Update the segment registers and DR7 from the CPU.
11290 */
11291 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11292 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11293 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11294 AssertRCReturn(rc, rc);
11295 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11296
11297 PVM pVM = pVCpu->CTX_SUFF(pVM);
11298 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11299 {
11300 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11301 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11302 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11303 if (RT_SUCCESS(rc))
11304 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11305 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11306 }
11307 else
11308 {
11309 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11310 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11311 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11313 }
11314
11315 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11316 if (RT_SUCCESS(rc))
11317 {
11318 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11319 AssertRCReturn(rc2, rc2);
11320 }
11321 return rc;
11322}
11323
11324
11325/**
11326 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11327 * Conditional VM-exit.
11328 */
11329HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11330{
11331 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11332 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11333
11334 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11335 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11336 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11337 return VINF_SUCCESS;
11338 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11339 return rc;
11340
11341 RTGCPHYS GCPhys = 0;
11342 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11343
11344#if 0
11345 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11346#else
11347 /* Aggressive state sync. for now. */
11348 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11349 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11350 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11351#endif
11352 AssertRCReturn(rc, rc);
11353
11354 /*
11355 * If we succeed, resume guest execution.
11356 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11357 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11358 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11359 * weird case. See @bugref{6043}.
11360 */
11361 PVM pVM = pVCpu->CTX_SUFF(pVM);
11362 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11363 rc = VBOXSTRICTRC_VAL(rc2);
11364 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11365 if ( rc == VINF_SUCCESS
11366 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11367 || rc == VERR_PAGE_NOT_PRESENT)
11368 {
11369 /* Successfully handled MMIO operation. */
11370 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11371 | HM_CHANGED_GUEST_RSP
11372 | HM_CHANGED_GUEST_RFLAGS
11373 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11374 rc = VINF_SUCCESS;
11375 }
11376 return rc;
11377}
11378
11379
11380/**
11381 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11382 * VM-exit.
11383 */
11384HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11385{
11386 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11387 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11388
11389 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11390 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11391 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11392 return VINF_SUCCESS;
11393 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11394 return rc;
11395
11396 RTGCPHYS GCPhys = 0;
11397 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11398 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11399#if 0
11400 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11401#else
11402 /* Aggressive state sync. for now. */
11403 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11404 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11405 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11406#endif
11407 AssertRCReturn(rc, rc);
11408
11409 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11410 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11411
11412 RTGCUINT uErrorCode = 0;
11413 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11414 uErrorCode |= X86_TRAP_PF_ID;
11415 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11416 uErrorCode |= X86_TRAP_PF_RW;
11417 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11418 uErrorCode |= X86_TRAP_PF_P;
11419
11420 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11421
11422 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11423 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11424
11425 /* Handle the pagefault trap for the nested shadow table. */
11426 PVM pVM = pVCpu->CTX_SUFF(pVM);
11427 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11428 TRPMResetTrap(pVCpu);
11429
11430 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11431 if ( rc == VINF_SUCCESS
11432 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11433 || rc == VERR_PAGE_NOT_PRESENT)
11434 {
11435 /* Successfully synced our nested page tables. */
11436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11437 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11438 | HM_CHANGED_GUEST_RSP
11439 | HM_CHANGED_GUEST_RFLAGS);
11440 return VINF_SUCCESS;
11441 }
11442
11443 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11444 return rc;
11445}
11446
11447/** @} */
11448
11449/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11450/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11451/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11452
11453/** @name VM-exit exception handlers.
11454 * @{
11455 */
11456
11457/**
11458 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11459 */
11460static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11461{
11462 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11464
11465 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11466 AssertRCReturn(rc, rc);
11467
11468 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11469 {
11470 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11471 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11472
11473 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11474 * provides VM-exit instruction length. If this causes problem later,
11475 * disassemble the instruction like it's done on AMD-V. */
11476 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11477 AssertRCReturn(rc2, rc2);
11478 return rc;
11479 }
11480
11481 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11482 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11483 return rc;
11484}
11485
11486
11487/**
11488 * VM-exit exception handler for #BP (Breakpoint exception).
11489 */
11490static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11491{
11492 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11493 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11494
11495 /** @todo Try optimize this by not saving the entire guest state unless
11496 * really needed. */
11497 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11498 AssertRCReturn(rc, rc);
11499
11500 PVM pVM = pVCpu->CTX_SUFF(pVM);
11501 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11502 if (rc == VINF_EM_RAW_GUEST_TRAP)
11503 {
11504 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11505 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11506 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11507 AssertRCReturn(rc, rc);
11508
11509 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11510 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11511 }
11512
11513 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11514 return rc;
11515}
11516
11517
11518/**
11519 * VM-exit exception handler for #DB (Debug exception).
11520 */
11521static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11522{
11523 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11524 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11525 Log6(("XcptDB\n"));
11526
11527 /*
11528 * Get the DR6-like values from the exit qualification and pass it to DBGF
11529 * for processing.
11530 */
11531 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11532 AssertRCReturn(rc, rc);
11533
11534 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11535 uint64_t uDR6 = X86_DR6_INIT_VAL;
11536 uDR6 |= ( pVmxTransient->uExitQualification
11537 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11538
11539 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11540 if (rc == VINF_EM_RAW_GUEST_TRAP)
11541 {
11542 /*
11543 * The exception was for the guest. Update DR6, DR7.GD and
11544 * IA32_DEBUGCTL.LBR before forwarding it.
11545 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11546 */
11547 VMMRZCallRing3Disable(pVCpu);
11548 HM_DISABLE_PREEMPT_IF_NEEDED();
11549
11550 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11551 pMixedCtx->dr[6] |= uDR6;
11552 if (CPUMIsGuestDebugStateActive(pVCpu))
11553 ASMSetDR6(pMixedCtx->dr[6]);
11554
11555 HM_RESTORE_PREEMPT_IF_NEEDED();
11556 VMMRZCallRing3Enable(pVCpu);
11557
11558 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11559 AssertRCReturn(rc, rc);
11560
11561 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11562 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11563
11564 /* Paranoia. */
11565 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11566 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11567
11568 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11569 AssertRCReturn(rc, rc);
11570
11571 /*
11572 * Raise #DB in the guest.
11573 *
11574 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11575 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11576 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11577 *
11578 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11579 */
11580 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11581 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11582 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11583 AssertRCReturn(rc, rc);
11584 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11585 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11586 return VINF_SUCCESS;
11587 }
11588
11589 /*
11590 * Not a guest trap, must be a hypervisor related debug event then.
11591 * Update DR6 in case someone is interested in it.
11592 */
11593 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11594 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11595 CPUMSetHyperDR6(pVCpu, uDR6);
11596
11597 return rc;
11598}
11599
11600
11601/**
11602 * VM-exit exception handler for #NM (Device-not-available exception: floating
11603 * point exception).
11604 */
11605static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11606{
11607 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11608
11609 /* We require CR0 and EFER. EFER is always up-to-date. */
11610 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11611 AssertRCReturn(rc, rc);
11612
11613 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11614 VMMRZCallRing3Disable(pVCpu);
11615 HM_DISABLE_PREEMPT_IF_NEEDED();
11616
11617 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11618 if (pVmxTransient->fWasGuestFPUStateActive)
11619 {
11620 rc = VINF_EM_RAW_GUEST_TRAP;
11621 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11622 }
11623 else
11624 {
11625#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11626 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11627#endif
11628 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11629 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11630 }
11631
11632 HM_RESTORE_PREEMPT_IF_NEEDED();
11633 VMMRZCallRing3Enable(pVCpu);
11634
11635 if (rc == VINF_SUCCESS)
11636 {
11637 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11638 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11639 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11640 pVCpu->hm.s.fUseGuestFpu = true;
11641 }
11642 else
11643 {
11644 /* Forward #NM to the guest. */
11645 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11646 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11647 AssertRCReturn(rc, rc);
11648 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11649 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11650 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11651 }
11652
11653 return VINF_SUCCESS;
11654}
11655
11656
11657/**
11658 * VM-exit exception handler for #GP (General-protection exception).
11659 *
11660 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11661 */
11662static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11666
11667 int rc = VERR_INTERNAL_ERROR_5;
11668 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11669 {
11670#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11671 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11672 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11673 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11674 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11675 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11676 AssertRCReturn(rc, rc);
11677 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11678 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11679 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11680 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11681 return rc;
11682#else
11683 /* We don't intercept #GP. */
11684 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11685 NOREF(pVmxTransient);
11686 return VERR_VMX_UNEXPECTED_EXCEPTION;
11687#endif
11688 }
11689
11690 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11691 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11692
11693 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11694 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11695 AssertRCReturn(rc, rc);
11696
11697 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11698 uint32_t cbOp = 0;
11699 PVM pVM = pVCpu->CTX_SUFF(pVM);
11700 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11701 if (RT_SUCCESS(rc))
11702 {
11703 rc = VINF_SUCCESS;
11704 Assert(cbOp == pDis->cbInstr);
11705 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11706 switch (pDis->pCurInstr->uOpcode)
11707 {
11708 case OP_CLI:
11709 {
11710 pMixedCtx->eflags.Bits.u1IF = 0;
11711 pMixedCtx->rip += pDis->cbInstr;
11712 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11713 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11715 break;
11716 }
11717
11718 case OP_STI:
11719 {
11720 pMixedCtx->eflags.Bits.u1IF = 1;
11721 pMixedCtx->rip += pDis->cbInstr;
11722 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11723 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11724 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11725 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11727 break;
11728 }
11729
11730 case OP_HLT:
11731 {
11732 rc = VINF_EM_HALT;
11733 pMixedCtx->rip += pDis->cbInstr;
11734 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11736 break;
11737 }
11738
11739 case OP_POPF:
11740 {
11741 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11742 uint32_t cbParm;
11743 uint32_t uMask;
11744 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11745 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11746 {
11747 cbParm = 4;
11748 uMask = 0xffffffff;
11749 }
11750 else
11751 {
11752 cbParm = 2;
11753 uMask = 0xffff;
11754 }
11755
11756 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11757 RTGCPTR GCPtrStack = 0;
11758 X86EFLAGS Eflags;
11759 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11760 &GCPtrStack);
11761 if (RT_SUCCESS(rc))
11762 {
11763 Assert(sizeof(Eflags.u32) >= cbParm);
11764 Eflags.u32 = 0;
11765 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11766 }
11767 if (RT_FAILURE(rc))
11768 {
11769 rc = VERR_EM_INTERPRETER;
11770 break;
11771 }
11772 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11773 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11774 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11775 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11776 pMixedCtx->esp += cbParm;
11777 pMixedCtx->esp &= uMask;
11778 pMixedCtx->rip += pDis->cbInstr;
11779 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11780 | HM_CHANGED_GUEST_RSP
11781 | HM_CHANGED_GUEST_RFLAGS);
11782 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11783 if (fStepping)
11784 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11785
11786 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11787 break;
11788 }
11789
11790 case OP_PUSHF:
11791 {
11792 uint32_t cbParm;
11793 uint32_t uMask;
11794 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11795 {
11796 cbParm = 4;
11797 uMask = 0xffffffff;
11798 }
11799 else
11800 {
11801 cbParm = 2;
11802 uMask = 0xffff;
11803 }
11804
11805 /* Get the stack pointer & push the contents of eflags onto the stack. */
11806 RTGCPTR GCPtrStack = 0;
11807 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11808 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11809 if (RT_FAILURE(rc))
11810 {
11811 rc = VERR_EM_INTERPRETER;
11812 break;
11813 }
11814 X86EFLAGS Eflags = pMixedCtx->eflags;
11815 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11816 Eflags.Bits.u1RF = 0;
11817 Eflags.Bits.u1VM = 0;
11818
11819 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11820 if (RT_FAILURE(rc))
11821 {
11822 rc = VERR_EM_INTERPRETER;
11823 break;
11824 }
11825 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11826 pMixedCtx->esp -= cbParm;
11827 pMixedCtx->esp &= uMask;
11828 pMixedCtx->rip += pDis->cbInstr;
11829 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11830 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11832 break;
11833 }
11834
11835 case OP_IRET:
11836 {
11837 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11838 * instruction reference. */
11839 RTGCPTR GCPtrStack = 0;
11840 uint32_t uMask = 0xffff;
11841 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11842 uint16_t aIretFrame[3];
11843 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11844 {
11845 rc = VERR_EM_INTERPRETER;
11846 break;
11847 }
11848 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11849 &GCPtrStack);
11850 if (RT_SUCCESS(rc))
11851 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11852 if (RT_FAILURE(rc))
11853 {
11854 rc = VERR_EM_INTERPRETER;
11855 break;
11856 }
11857 pMixedCtx->eip = 0;
11858 pMixedCtx->ip = aIretFrame[0];
11859 pMixedCtx->cs.Sel = aIretFrame[1];
11860 pMixedCtx->cs.ValidSel = aIretFrame[1];
11861 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11862 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11863 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11864 pMixedCtx->sp += sizeof(aIretFrame);
11865 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11866 | HM_CHANGED_GUEST_SEGMENT_REGS
11867 | HM_CHANGED_GUEST_RSP
11868 | HM_CHANGED_GUEST_RFLAGS);
11869 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11870 if (fStepping)
11871 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11872 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11874 break;
11875 }
11876
11877 case OP_INT:
11878 {
11879 uint16_t uVector = pDis->Param1.uValue & 0xff;
11880 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11881 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11883 break;
11884 }
11885
11886 case OP_INTO:
11887 {
11888 if (pMixedCtx->eflags.Bits.u1OF)
11889 {
11890 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11891 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11892 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11893 }
11894 break;
11895 }
11896
11897 default:
11898 {
11899 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11900 EMCODETYPE_SUPERVISOR);
11901 rc = VBOXSTRICTRC_VAL(rc2);
11902 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11903 /** @todo We have to set pending-debug exceptions here when the guest is
11904 * single-stepping depending on the instruction that was interpreted. */
11905 Log4(("#GP rc=%Rrc\n", rc));
11906 break;
11907 }
11908 }
11909 }
11910 else
11911 rc = VERR_EM_INTERPRETER;
11912
11913 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11914 ("#GP Unexpected rc=%Rrc\n", rc));
11915 return rc;
11916}
11917
11918
11919#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11920/**
11921 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11922 * the exception reported in the VMX transient structure back into the VM.
11923 *
11924 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11925 * up-to-date.
11926 */
11927static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11928{
11929 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11930
11931 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11932 hmR0VmxCheckExitDueToEventDelivery(). */
11933 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11934 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11935 AssertRCReturn(rc, rc);
11936 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11937
11938#ifdef DEBUG_ramshankar
11939 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11940 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11941 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11942#endif
11943
11944 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11945 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11946 return VINF_SUCCESS;
11947}
11948#endif
11949
11950
11951/**
11952 * VM-exit exception handler for #PF (Page-fault exception).
11953 */
11954static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11955{
11956 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11957 PVM pVM = pVCpu->CTX_SUFF(pVM);
11958 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11959 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11960 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11961 AssertRCReturn(rc, rc);
11962
11963#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11964 if (pVM->hm.s.fNestedPaging)
11965 {
11966 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11967 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11968 {
11969 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11970 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11971 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11972 }
11973 else
11974 {
11975 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11976 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11977 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11978 }
11979 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11980 return rc;
11981 }
11982#else
11983 Assert(!pVM->hm.s.fNestedPaging);
11984 NOREF(pVM);
11985#endif
11986
11987 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11988 AssertRCReturn(rc, rc);
11989
11990 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11991 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11992
11993 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11994 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11995 (RTGCPTR)pVmxTransient->uExitQualification);
11996
11997 Log4(("#PF: rc=%Rrc\n", rc));
11998 if (rc == VINF_SUCCESS)
11999 {
12000 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
12001 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
12002 * memory? We don't update the whole state here... */
12003 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
12004 | HM_CHANGED_GUEST_RSP
12005 | HM_CHANGED_GUEST_RFLAGS
12006 | HM_CHANGED_VMX_GUEST_APIC_STATE);
12007 TRPMResetTrap(pVCpu);
12008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
12009 return rc;
12010 }
12011 else if (rc == VINF_EM_RAW_GUEST_TRAP)
12012 {
12013 if (!pVmxTransient->fVectoringPF)
12014 {
12015 /* It's a guest page fault and needs to be reflected to the guest. */
12016 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
12017 TRPMResetTrap(pVCpu);
12018 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
12019 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
12020 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12021 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
12022 }
12023 else
12024 {
12025 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
12026 TRPMResetTrap(pVCpu);
12027 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
12028 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
12029 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
12030 }
12031
12032 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
12033 return VINF_SUCCESS;
12034 }
12035
12036 TRPMResetTrap(pVCpu);
12037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
12038 return rc;
12039}
12040
12041/** @} */
12042
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette