VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 51220

Last change on this file since 51220 was 51220, checked in by vboxsync, 11 years ago

VMM/HMVMXR0: Implemented EFER swapping using VMCS controls.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 487.4 KB
Line 
1/* $Id: HMVMXR0.cpp 51220 2014-05-09 01:51:16Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2013 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18/*******************************************************************************
19* Header Files *
20*******************************************************************************/
21#define LOG_GROUP LOG_GROUP_HM
22#include <iprt/x86.h>
23#include <iprt/asm-amd64-x86.h>
24#include <iprt/thread.h>
25#include <iprt/string.h>
26
27#include "HMInternal.h"
28#include <VBox/vmm/vm.h>
29#include "HMVMXR0.h"
30#include <VBox/vmm/pdmapi.h>
31#include <VBox/vmm/dbgf.h>
32#include <VBox/vmm/iem.h>
33#include <VBox/vmm/iom.h>
34#include <VBox/vmm/selm.h>
35#include <VBox/vmm/tm.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#ifdef DEBUG_ramshankar
40# define HMVMX_SAVE_FULL_GUEST_STATE
41# define HMVMX_SYNC_FULL_GUEST_STATE
42# define HMVMX_ALWAYS_CHECK_GUEST_STATE
43# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
44# define HMVMX_ALWAYS_TRAP_PF
45# define HMVMX_ALWAYS_SWAP_FPU_STATE
46# define HMVMX_ALWAYS_FLUSH_TLB
47#endif
48
49
50/*******************************************************************************
51* Defined Constants And Macros *
52*******************************************************************************/
53#if defined(RT_ARCH_AMD64)
54# define HMVMX_IS_64BIT_HOST_MODE() (true)
55typedef RTHCUINTREG HMVMXHCUINTREG;
56#elif defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
57extern "C" uint32_t g_fVMXIs64bitHost;
58# define HMVMX_IS_64BIT_HOST_MODE() (g_fVMXIs64bitHost != 0)
59typedef uint64_t HMVMXHCUINTREG;
60#else
61# define HMVMX_IS_64BIT_HOST_MODE() (false)
62typedef RTHCUINTREG HMVMXHCUINTREG;
63#endif
64
65/** Use the function table. */
66#define HMVMX_USE_FUNCTION_TABLE
67
68/** Determine which tagged-TLB flush handler to use. */
69#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
70#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
71#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
72#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
73
74/** @name Updated-guest-state flags.
75 * @{ */
76#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
77#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
78#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
79#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
80#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
81#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
82#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
83#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
84#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
85#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
86#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
87#define HMVMX_UPDATED_GUEST_DEBUG RT_BIT(11)
88#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
89#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
90#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
91#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
92#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
93#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
94#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(18)
95#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
96 | HMVMX_UPDATED_GUEST_RSP \
97 | HMVMX_UPDATED_GUEST_RFLAGS \
98 | HMVMX_UPDATED_GUEST_CR0 \
99 | HMVMX_UPDATED_GUEST_CR3 \
100 | HMVMX_UPDATED_GUEST_CR4 \
101 | HMVMX_UPDATED_GUEST_GDTR \
102 | HMVMX_UPDATED_GUEST_IDTR \
103 | HMVMX_UPDATED_GUEST_LDTR \
104 | HMVMX_UPDATED_GUEST_TR \
105 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
106 | HMVMX_UPDATED_GUEST_DEBUG \
107 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
108 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
109 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
110 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
111 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
112 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
113 | HMVMX_UPDATED_GUEST_APIC_STATE)
114/** @} */
115
116/** @name
117 * Flags to skip redundant reads of some common VMCS fields that are not part of
118 * the guest-CPU state but are in the transient structure.
119 */
120#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
121#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
123#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
124#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
125#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
126#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
127/** @} */
128
129/** @name
130 * States of the VMCS.
131 *
132 * This does not reflect all possible VMCS states but currently only those
133 * needed for maintaining the VMCS consistently even when thread-context hooks
134 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
135 */
136#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
137#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
138#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
139/** @} */
140
141/**
142 * Exception bitmap mask for real-mode guests (real-on-v86).
143 *
144 * We need to intercept all exceptions manually (except #PF). #NM is also
145 * handled separately, see hmR0VmxLoadSharedCR0(). #PF need not be intercepted
146 * even in real-mode if we have Nested Paging support.
147 */
148#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) | RT_BIT(X86_XCPT_DB) | RT_BIT(X86_XCPT_NMI) \
149 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
150 | RT_BIT(X86_XCPT_UD) /* RT_BIT(X86_XCPT_NM) */ | RT_BIT(X86_XCPT_DF) \
151 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
152 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
153 | RT_BIT(X86_XCPT_MF) | RT_BIT(X86_XCPT_AC) | RT_BIT(X86_XCPT_MC) \
154 | RT_BIT(X86_XCPT_XF))
155
156/**
157 * Exception bitmap mask for all contributory exceptions.
158 *
159 * Page fault is deliberately excluded here as it's conditional as to whether
160 * it's contributory or benign. Page faults are handled separately.
161 */
162#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
163 | RT_BIT(X86_XCPT_DE))
164
165/** Maximum VM-instruction error number. */
166#define HMVMX_INSTR_ERROR_MAX 28
167
168/** Profiling macro. */
169#ifdef HM_PROFILE_EXIT_DISPATCH
170# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
171# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
172#else
173# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
174# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
175#endif
176
177/** Assert that preemption is disabled or covered by thread-context hooks. */
178#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
179 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
180
181/** Assert that we haven't migrated CPUs when thread-context hooks are not
182 * used. */
183#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHooksAreRegistered(pVCpu) \
184 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
185 ("Illegal migration! Entered on CPU %u Current %u\n", \
186 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
187
188/** Helper macro for VM-exit handlers called unexpectedly. */
189#define HMVMX_RETURN_UNEXPECTED_EXIT() \
190 do { \
191 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
192 return VERR_VMX_UNEXPECTED_EXIT; \
193 } while (0)
194
195
196/*******************************************************************************
197* Structures and Typedefs *
198*******************************************************************************/
199/**
200 * VMX transient state.
201 *
202 * A state structure for holding miscellaneous information across
203 * VMX non-root operation and restored after the transition.
204 */
205typedef struct VMXTRANSIENT
206{
207 /** The host's rflags/eflags. */
208 RTCCUINTREG uEflags;
209#if HC_ARCH_BITS == 32
210 uint32_t u32Alignment0;
211#endif
212 /** The guest's TPR value used for TPR shadowing. */
213 uint8_t u8GuestTpr;
214 /** Alignment. */
215 uint8_t abAlignment0[7];
216
217 /** The basic VM-exit reason. */
218 uint16_t uExitReason;
219 /** Alignment. */
220 uint16_t u16Alignment0;
221 /** The VM-exit interruption error code. */
222 uint32_t uExitIntErrorCode;
223 /** The VM-exit exit qualification. */
224 uint64_t uExitQualification;
225
226 /** The VM-exit interruption-information field. */
227 uint32_t uExitIntInfo;
228 /** The VM-exit instruction-length field. */
229 uint32_t cbInstr;
230 /** The VM-exit instruction-information field. */
231 union
232 {
233 /** Plain unsigned int representation. */
234 uint32_t u;
235 /** INS and OUTS information. */
236 struct
237 {
238 uint32_t u6Reserved0 : 7;
239 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
240 uint32_t u3AddrSize : 3;
241 uint32_t u5Reserved1 : 5;
242 /** The segment register (X86_SREG_XXX). */
243 uint32_t iSegReg : 3;
244 uint32_t uReserved2 : 14;
245 } StrIo;
246 } ExitInstrInfo;
247 /** Whether the VM-entry failed or not. */
248 bool fVMEntryFailed;
249 /** Alignment. */
250 uint8_t abAlignment1[3];
251
252 /** The VM-entry interruption-information field. */
253 uint32_t uEntryIntInfo;
254 /** The VM-entry exception error code field. */
255 uint32_t uEntryXcptErrorCode;
256 /** The VM-entry instruction length field. */
257 uint32_t cbEntryInstr;
258
259 /** IDT-vectoring information field. */
260 uint32_t uIdtVectoringInfo;
261 /** IDT-vectoring error code. */
262 uint32_t uIdtVectoringErrorCode;
263
264 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
265 uint32_t fVmcsFieldsRead;
266
267 /** Whether the guest FPU was active at the time of VM-exit. */
268 bool fWasGuestFPUStateActive;
269 /** Whether the guest debug state was active at the time of VM-exit. */
270 bool fWasGuestDebugStateActive;
271 /** Whether the hyper debug state was active at the time of VM-exit. */
272 bool fWasHyperDebugStateActive;
273 /** Whether TSC-offsetting should be setup before VM-entry. */
274 bool fUpdateTscOffsettingAndPreemptTimer;
275 /** Whether the VM-exit was caused by a page-fault during delivery of a
276 * contributory exception or a page-fault. */
277 bool fVectoringPF;
278} VMXTRANSIENT;
279AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
280AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
281AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
282AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestFPUStateActive, sizeof(uint64_t));
283AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
284/** Pointer to VMX transient state. */
285typedef VMXTRANSIENT *PVMXTRANSIENT;
286
287
288/**
289 * MSR-bitmap read permissions.
290 */
291typedef enum VMXMSREXITREAD
292{
293 /** Reading this MSR causes a VM-exit. */
294 VMXMSREXIT_INTERCEPT_READ = 0xb,
295 /** Reading this MSR does not cause a VM-exit. */
296 VMXMSREXIT_PASSTHRU_READ
297} VMXMSREXITREAD;
298/** Pointer to MSR-bitmap read permissions. */
299typedef VMXMSREXITREAD* PVMXMSREXITREAD;
300
301/**
302 * MSR-bitmap write permissions.
303 */
304typedef enum VMXMSREXITWRITE
305{
306 /** Writing to this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
308 /** Writing to this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_WRITE
310} VMXMSREXITWRITE;
311/** Pointer to MSR-bitmap write permissions. */
312typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
313
314
315/**
316 * VMX VM-exit handler.
317 *
318 * @returns VBox status code.
319 * @param pVCpu Pointer to the VMCPU.
320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
321 * out-of-sync. Make sure to update the required
322 * fields before using them.
323 * @param pVmxTransient Pointer to the VMX-transient structure.
324 */
325#ifndef HMVMX_USE_FUNCTION_TABLE
326typedef int FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
327#else
328typedef DECLCALLBACK(int) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
329/** Pointer to VM-exit handler. */
330typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
331#endif
332
333
334/*******************************************************************************
335* Internal Functions *
336*******************************************************************************/
337static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush);
338static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr);
339static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
340 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntState);
341#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
342static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
343#endif
344#ifndef HMVMX_USE_FUNCTION_TABLE
345DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
346# define HMVMX_EXIT_DECL static int
347#else
348# define HMVMX_EXIT_DECL static DECLCALLBACK(int)
349#endif
350
351/** @name VM-exit handlers.
352 * @{
353 */
354static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
355static FNVMXEXITHANDLER hmR0VmxExitExtInt;
356static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
357static FNVMXEXITHANDLER hmR0VmxExitInitSignal;
358static FNVMXEXITHANDLER hmR0VmxExitSipi;
359static FNVMXEXITHANDLER hmR0VmxExitIoSmi;
360static FNVMXEXITHANDLER hmR0VmxExitSmi;
361static FNVMXEXITHANDLER hmR0VmxExitIntWindow;
362static FNVMXEXITHANDLER hmR0VmxExitNmiWindow;
363static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
364static FNVMXEXITHANDLER hmR0VmxExitCpuid;
365static FNVMXEXITHANDLER hmR0VmxExitGetsec;
366static FNVMXEXITHANDLER hmR0VmxExitHlt;
367static FNVMXEXITHANDLER hmR0VmxExitInvd;
368static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
369static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
370static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
371static FNVMXEXITHANDLER hmR0VmxExitRsm;
372static FNVMXEXITHANDLER hmR0VmxExitSetPendingXcptUD;
373static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
374static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
375static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
376static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
377static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
378static FNVMXEXITHANDLER hmR0VmxExitErrInvalidGuestState;
379static FNVMXEXITHANDLER hmR0VmxExitErrMsrLoad;
380static FNVMXEXITHANDLER hmR0VmxExitErrUndefined;
381static FNVMXEXITHANDLER hmR0VmxExitMwait;
382static FNVMXEXITHANDLER hmR0VmxExitMtf;
383static FNVMXEXITHANDLER hmR0VmxExitMonitor;
384static FNVMXEXITHANDLER hmR0VmxExitPause;
385static FNVMXEXITHANDLER hmR0VmxExitErrMachineCheck;
386static FNVMXEXITHANDLER hmR0VmxExitTprBelowThreshold;
387static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
388static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
389static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
390static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
391static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
392static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
393static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
394static FNVMXEXITHANDLER hmR0VmxExitWbinvd;
395static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
396static FNVMXEXITHANDLER hmR0VmxExitRdrand;
397static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
398/** @} */
399
400static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
401static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
402static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
403static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
404static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
405static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
406#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
407static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
408#endif
409static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
410
411/*******************************************************************************
412* Global Variables *
413*******************************************************************************/
414#ifdef HMVMX_USE_FUNCTION_TABLE
415
416/**
417 * VMX_EXIT dispatch table.
418 */
419static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
420{
421 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
422 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
423 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
424 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
425 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
426 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
427 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
428 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
429 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
430 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
431 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
432 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
433 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
434 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
435 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
436 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
437 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
438 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
439 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitSetPendingXcptUD,
440 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
441 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
442 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
443 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
444 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
445 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
446 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
447 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
448 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
449 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
450 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
451 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
452 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
453 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
454 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
455 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
456 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
457 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
458 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
459 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
460 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
461 /* 40 UNDEFINED */ hmR0VmxExitPause,
462 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
463 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
464 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
465 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
466 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
467 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
468 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
469 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
470 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
471 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
472 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
473 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
474 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
475 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
476 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
477 /* 56 UNDEFINED */ hmR0VmxExitErrUndefined,
478 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
479 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
480 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD
481};
482#endif /* HMVMX_USE_FUNCTION_TABLE */
483
484#ifdef VBOX_STRICT
485static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
486{
487 /* 0 */ "(Not Used)",
488 /* 1 */ "VMCALL executed in VMX root operation.",
489 /* 2 */ "VMCLEAR with invalid physical address.",
490 /* 3 */ "VMCLEAR with VMXON pointer.",
491 /* 4 */ "VMLAUNCH with non-clear VMCS.",
492 /* 5 */ "VMRESUME with non-launched VMCS.",
493 /* 6 */ "VMRESUME after VMXOFF",
494 /* 7 */ "VM entry with invalid control fields.",
495 /* 8 */ "VM entry with invalid host state fields.",
496 /* 9 */ "VMPTRLD with invalid physical address.",
497 /* 10 */ "VMPTRLD with VMXON pointer.",
498 /* 11 */ "VMPTRLD with incorrect revision identifier.",
499 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
500 /* 13 */ "VMWRITE to read-only VMCS component.",
501 /* 14 */ "(Not Used)",
502 /* 15 */ "VMXON executed in VMX root operation.",
503 /* 16 */ "VM entry with invalid executive-VMCS pointer.",
504 /* 17 */ "VM entry with non-launched executing VMCS.",
505 /* 18 */ "VM entry with executive-VMCS pointer not VMXON pointer.",
506 /* 19 */ "VMCALL with non-clear VMCS.",
507 /* 20 */ "VMCALL with invalid VM-exit control fields.",
508 /* 21 */ "(Not Used)",
509 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
510 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
511 /* 24 */ "VMCALL with invalid SMM-monitor features.",
512 /* 25 */ "VM entry with invalid VM-execution control fields in executive VMCS.",
513 /* 26 */ "VM entry with events blocked by MOV SS.",
514 /* 27 */ "(Not Used)",
515 /* 28 */ "Invalid operand to INVEPT/INVVPID."
516};
517#endif /* VBOX_STRICT */
518
519
520
521/**
522 * Updates the VM's last error record. If there was a VMX instruction error,
523 * reads the error data from the VMCS and updates VCPU's last error record as
524 * well.
525 *
526 * @param pVM Pointer to the VM.
527 * @param pVCpu Pointer to the VMCPU (can be NULL if @a rc is not
528 * VERR_VMX_UNABLE_TO_START_VM or
529 * VERR_VMX_INVALID_VMCS_FIELD).
530 * @param rc The error code.
531 */
532static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
533{
534 AssertPtr(pVM);
535 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
536 || rc == VERR_VMX_UNABLE_TO_START_VM)
537 {
538 AssertPtrReturnVoid(pVCpu);
539 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
540 }
541 pVM->hm.s.lLastError = rc;
542}
543
544
545/**
546 * Reads the VM-entry interruption-information field from the VMCS into the VMX
547 * transient structure.
548 *
549 * @returns VBox status code.
550 * @param pVmxTransient Pointer to the VMX transient structure.
551 *
552 * @remarks No-long-jump zone!!!
553 */
554DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
555{
556 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
557 AssertRCReturn(rc, rc);
558 return VINF_SUCCESS;
559}
560
561
562/**
563 * Reads the VM-entry exception error code field from the VMCS into
564 * the VMX transient structure.
565 *
566 * @returns VBox status code.
567 * @param pVmxTransient Pointer to the VMX transient structure.
568 *
569 * @remarks No-long-jump zone!!!
570 */
571DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
572{
573 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
574 AssertRCReturn(rc, rc);
575 return VINF_SUCCESS;
576}
577
578
579/**
580 * Reads the VM-entry exception error code field from the VMCS into
581 * the VMX transient structure.
582 *
583 * @returns VBox status code.
584 * @param pVmxTransient Pointer to the VMX transient structure.
585 *
586 * @remarks No-long-jump zone!!!
587 */
588DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
589{
590 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
591 AssertRCReturn(rc, rc);
592 return VINF_SUCCESS;
593}
594
595
596/**
597 * Reads the VM-exit interruption-information field from the VMCS into the VMX
598 * transient structure.
599 *
600 * @returns VBox status code.
601 * @param pVmxTransient Pointer to the VMX transient structure.
602 */
603DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
604{
605 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
606 {
607 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
608 AssertRCReturn(rc, rc);
609 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
610 }
611 return VINF_SUCCESS;
612}
613
614
615/**
616 * Reads the VM-exit interruption error code from the VMCS into the VMX
617 * transient structure.
618 *
619 * @returns VBox status code.
620 * @param pVmxTransient Pointer to the VMX transient structure.
621 */
622DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
623{
624 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
625 {
626 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
627 AssertRCReturn(rc, rc);
628 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
629 }
630 return VINF_SUCCESS;
631}
632
633
634/**
635 * Reads the VM-exit instruction length field from the VMCS into the VMX
636 * transient structure.
637 *
638 * @returns VBox status code.
639 * @param pVCpu Pointer to the VMCPU.
640 * @param pVmxTransient Pointer to the VMX transient structure.
641 */
642DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
643{
644 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
645 {
646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
647 AssertRCReturn(rc, rc);
648 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
649 }
650 return VINF_SUCCESS;
651}
652
653
654/**
655 * Reads the VM-exit instruction-information field from the VMCS into
656 * the VMX transient structure.
657 *
658 * @returns VBox status code.
659 * @param pVmxTransient Pointer to the VMX transient structure.
660 */
661DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
662{
663 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
664 {
665 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
666 AssertRCReturn(rc, rc);
667 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
668 }
669 return VINF_SUCCESS;
670}
671
672
673/**
674 * Reads the exit qualification from the VMCS into the VMX transient structure.
675 *
676 * @returns VBox status code.
677 * @param pVCpu Pointer to the VMCPU (required for the VMCS cache
678 * case).
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
684 {
685 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the IDT-vectoring information field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 *
700 * @remarks No-long-jump zone!!!
701 */
702DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
703{
704 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
705 {
706 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
707 AssertRCReturn(rc, rc);
708 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
709 }
710 return VINF_SUCCESS;
711}
712
713
714/**
715 * Reads the IDT-vectoring error code from the VMCS into the VMX
716 * transient structure.
717 *
718 * @returns VBox status code.
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
724 {
725 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Enters VMX root mode operation on the current CPU.
735 *
736 * @returns VBox status code.
737 * @param pVM Pointer to the VM (optional, can be NULL, after
738 * a resume).
739 * @param HCPhysCpuPage Physical address of the VMXON region.
740 * @param pvCpuPage Pointer to the VMXON region.
741 */
742static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
743{
744 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
745 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
746 Assert(pvCpuPage);
747 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
748
749 if (pVM)
750 {
751 /* Write the VMCS revision dword to the VMXON region. */
752 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
753 }
754
755 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
756 RTCCUINTREG uEflags = ASMIntDisableFlags();
757
758 /* Enable the VMX bit in CR4 if necessary. */
759 RTCCUINTREG uCr4 = ASMGetCR4();
760 if (!(uCr4 & X86_CR4_VMXE))
761 ASMSetCR4(uCr4 | X86_CR4_VMXE);
762
763 /* Enter VMX root mode. */
764 int rc = VMXEnable(HCPhysCpuPage);
765 if (RT_FAILURE(rc))
766 ASMSetCR4(uCr4);
767
768 /* Restore interrupts. */
769 ASMSetFlags(uEflags);
770 return rc;
771}
772
773
774/**
775 * Exits VMX root mode operation on the current CPU.
776 *
777 * @returns VBox status code.
778 */
779static int hmR0VmxLeaveRootMode(void)
780{
781 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
782
783 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
784 RTCCUINTREG uEflags = ASMIntDisableFlags();
785
786 /* If we're for some reason not in VMX root mode, then don't leave it. */
787 RTCCUINTREG uHostCR4 = ASMGetCR4();
788
789 int rc;
790 if (uHostCR4 & X86_CR4_VMXE)
791 {
792 /* Exit VMX root mode and clear the VMX bit in CR4. */
793 VMXDisable();
794 ASMSetCR4(uHostCR4 & ~X86_CR4_VMXE);
795 rc = VINF_SUCCESS;
796 }
797 else
798 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
799
800 /* Restore interrupts. */
801 ASMSetFlags(uEflags);
802 return rc;
803}
804
805
806/**
807 * Allocates and maps one physically contiguous page. The allocated page is
808 * zero'd out. (Used by various VT-x structures).
809 *
810 * @returns IPRT status code.
811 * @param pMemObj Pointer to the ring-0 memory object.
812 * @param ppVirt Where to store the virtual address of the
813 * allocation.
814 * @param pPhys Where to store the physical address of the
815 * allocation.
816 */
817DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
818{
819 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
820 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
821 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
822
823 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
824 if (RT_FAILURE(rc))
825 return rc;
826 *ppVirt = RTR0MemObjAddress(*pMemObj);
827 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
828 ASMMemZero32(*ppVirt, PAGE_SIZE);
829 return VINF_SUCCESS;
830}
831
832
833/**
834 * Frees and unmaps an allocated physical page.
835 *
836 * @param pMemObj Pointer to the ring-0 memory object.
837 * @param ppVirt Where to re-initialize the virtual address of
838 * allocation as 0.
839 * @param pHCPhys Where to re-initialize the physical address of the
840 * allocation as 0.
841 */
842DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
843{
844 AssertPtr(pMemObj);
845 AssertPtr(ppVirt);
846 AssertPtr(pHCPhys);
847 if (*pMemObj != NIL_RTR0MEMOBJ)
848 {
849 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
850 AssertRC(rc);
851 *pMemObj = NIL_RTR0MEMOBJ;
852 *ppVirt = 0;
853 *pHCPhys = 0;
854 }
855}
856
857
858/**
859 * Worker function to free VT-x related structures.
860 *
861 * @returns IPRT status code.
862 * @param pVM Pointer to the VM.
863 */
864static void hmR0VmxStructsFree(PVM pVM)
865{
866 for (VMCPUID i = 0; i < pVM->cCpus; i++)
867 {
868 PVMCPU pVCpu = &pVM->aCpus[i];
869 AssertPtr(pVCpu);
870
871 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
872 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
873
874 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
875 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
876
877 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic, &pVCpu->hm.s.vmx.HCPhysVirtApic);
878 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
879 }
880
881 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
882#ifdef VBOX_WITH_CRASHDUMP_MAGIC
883 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
884#endif
885}
886
887
888/**
889 * Worker function to allocate VT-x related VM structures.
890 *
891 * @returns IPRT status code.
892 * @param pVM Pointer to the VM.
893 */
894static int hmR0VmxStructsAlloc(PVM pVM)
895{
896 /*
897 * Initialize members up-front so we can cleanup properly on allocation failure.
898 */
899#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
900 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
901 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
902 pVM->hm.s.vmx.HCPhys##a_Name = 0;
903
904#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
905 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
906 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
907 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
908
909#ifdef VBOX_WITH_CRASHDUMP_MAGIC
910 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
911#endif
912 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
913
914 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
919 VMXLOCAL_INIT_VMCPU_MEMOBJ(VirtApic, pb);
920 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
921 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
922 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
923 }
924#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
925#undef VMXLOCAL_INIT_VM_MEMOBJ
926
927 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
928 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
929 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
930 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
931
932 /*
933 * Allocate all the VT-x structures.
934 */
935 int rc = VINF_SUCCESS;
936#ifdef VBOX_WITH_CRASHDUMP_MAGIC
937 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
938 if (RT_FAILURE(rc))
939 goto cleanup;
940 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
941 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
942#endif
943
944 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
945 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
946 {
947 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
948 &pVM->hm.s.vmx.HCPhysApicAccess);
949 if (RT_FAILURE(rc))
950 goto cleanup;
951 }
952
953 /*
954 * Initialize per-VCPU VT-x structures.
955 */
956 for (VMCPUID i = 0; i < pVM->cCpus; i++)
957 {
958 PVMCPU pVCpu = &pVM->aCpus[i];
959 AssertPtr(pVCpu);
960
961 /* Allocate the VM control structure (VMCS). */
962 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
963 if (RT_FAILURE(rc))
964 goto cleanup;
965
966 /* Allocate the Virtual-APIC page for transparent TPR accesses. */
967 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
968 {
969 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
970 &pVCpu->hm.s.vmx.HCPhysVirtApic);
971 if (RT_FAILURE(rc))
972 goto cleanup;
973 }
974
975 /* Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for transparent accesses of specific MSRs. */
976 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
977 {
978 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
979 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
983 }
984
985 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
986 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989
990 /* Allocate the VM-exit MSR-load page for the host MSRs. */
991 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
992 if (RT_FAILURE(rc))
993 goto cleanup;
994 }
995
996 return VINF_SUCCESS;
997
998cleanup:
999 hmR0VmxStructsFree(pVM);
1000 return rc;
1001}
1002
1003
1004/**
1005 * Does global VT-x initialization (called during module initialization).
1006 *
1007 * @returns VBox status code.
1008 */
1009VMMR0DECL(int) VMXR0GlobalInit(void)
1010{
1011#ifdef HMVMX_USE_FUNCTION_TABLE
1012 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1013# ifdef VBOX_STRICT
1014 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1015 Assert(g_apfnVMExitHandlers[i]);
1016# endif
1017#endif
1018 return VINF_SUCCESS;
1019}
1020
1021
1022/**
1023 * Does global VT-x termination (called during module termination).
1024 */
1025VMMR0DECL(void) VMXR0GlobalTerm()
1026{
1027 /* Nothing to do currently. */
1028}
1029
1030
1031/**
1032 * Sets up and activates VT-x on the current CPU.
1033 *
1034 * @returns VBox status code.
1035 * @param pCpu Pointer to the global CPU info struct.
1036 * @param pVM Pointer to the VM (can be NULL after a host resume
1037 * operation).
1038 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1039 * fEnabledByHost is true).
1040 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1041 * @a fEnabledByHost is true).
1042 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1043 * enable VT-x on the host.
1044 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1045 */
1046VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1047 void *pvMsrs)
1048{
1049 Assert(pCpu);
1050 Assert(pvMsrs);
1051 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1052
1053 /* Enable VT-x if it's not already enabled by the host. */
1054 if (!fEnabledByHost)
1055 {
1056 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1057 if (RT_FAILURE(rc))
1058 return rc;
1059 }
1060
1061 /*
1062 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1063 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1064 */
1065 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1066 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1067 {
1068 hmR0VmxFlushEpt(NULL /* pVCpu */, VMX_FLUSH_EPT_ALL_CONTEXTS);
1069 pCpu->fFlushAsidBeforeUse = false;
1070 }
1071 else
1072 pCpu->fFlushAsidBeforeUse = true;
1073
1074 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1075 ++pCpu->cTlbFlushes;
1076
1077 return VINF_SUCCESS;
1078}
1079
1080
1081/**
1082 * Deactivates VT-x on the current CPU.
1083 *
1084 * @returns VBox status code.
1085 * @param pCpu Pointer to the global CPU info struct.
1086 * @param pvCpuPage Pointer to the VMXON region.
1087 * @param HCPhysCpuPage Physical address of the VMXON region.
1088 *
1089 * @remarks This function should never be called when SUPR0EnableVTx() or
1090 * similar was used to enable VT-x on the host.
1091 */
1092VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1093{
1094 NOREF(pCpu);
1095 NOREF(pvCpuPage);
1096 NOREF(HCPhysCpuPage);
1097
1098 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1099 return hmR0VmxLeaveRootMode();
1100}
1101
1102
1103/**
1104 * Sets the permission bits for the specified MSR in the MSR bitmap.
1105 *
1106 * @param pVCpu Pointer to the VMCPU.
1107 * @param uMSR The MSR value.
1108 * @param enmRead Whether reading this MSR causes a VM-exit.
1109 * @param enmWrite Whether writing this MSR causes a VM-exit.
1110 */
1111static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1112{
1113 int32_t iBit;
1114 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1115
1116 /*
1117 * Layout:
1118 * 0x000 - 0x3ff - Low MSR read bits
1119 * 0x400 - 0x7ff - High MSR read bits
1120 * 0x800 - 0xbff - Low MSR write bits
1121 * 0xc00 - 0xfff - High MSR write bits
1122 */
1123 if (uMsr <= 0x00001FFF)
1124 iBit = uMsr;
1125 else if ( uMsr >= 0xC0000000
1126 && uMsr <= 0xC0001FFF)
1127 {
1128 iBit = (uMsr - 0xC0000000);
1129 pbMsrBitmap += 0x400;
1130 }
1131 else
1132 {
1133 AssertMsgFailed(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1134 return;
1135 }
1136
1137 Assert(iBit <= 0x1fff);
1138 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1139 ASMBitSet(pbMsrBitmap, iBit);
1140 else
1141 ASMBitClear(pbMsrBitmap, iBit);
1142
1143 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1144 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1145 else
1146 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1147}
1148
1149
1150#ifdef VBOX_STRICT
1151/**
1152 * Gets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @returns VBox status code.
1155 * @retval VINF_SUCCESS if the specified MSR is found.
1156 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1157 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1158 *
1159 * @param pVCpu Pointer to the VMCPU.
1160 * @param uMsr The MSR.
1161 * @param penmRead Where to store the read permissions.
1162 * @param penmWrite Where to store the write permissions.
1163 */
1164static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1165{
1166 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1167 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1168 int32_t iBit;
1169 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171 /* See hmR0VmxSetMsrPermission() for the layout. */
1172 if (uMsr <= 0x00001FFF)
1173 iBit = uMsr;
1174 else if ( uMsr >= 0xC0000000
1175 && uMsr <= 0xC0001FFF)
1176 {
1177 iBit = (uMsr - 0xC0000000);
1178 pbMsrBitmap += 0x400;
1179 }
1180 else
1181 {
1182 AssertMsgFailed(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1183 return VERR_NOT_SUPPORTED;
1184 }
1185
1186 Assert(iBit <= 0x1fff);
1187 if (ASMBitTest(pbMsrBitmap, iBit))
1188 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1189 else
1190 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1191
1192 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1193 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1194 else
1195 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1196 return VINF_SUCCESS;
1197}
1198#endif /* VBOX_STRICT */
1199
1200
1201/**
1202 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1203 * area.
1204 *
1205 * @returns VBox status code.
1206 * @param pVCpu Pointer to the VMCPU.
1207 * @param cMsrs The number of MSRs.
1208 */
1209DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1210{
1211 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1212 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1213 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1214 {
1215 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1216 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1218 }
1219
1220 /* Update number of guest MSRs to load/store across the world-switch. */
1221 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1222 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs); AssertRCReturn(rc, rc);
1223
1224 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1225 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs); AssertRCReturn(rc, rc);
1226
1227 /* Update the VCPU's copy of the MSR count. */
1228 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1229
1230 return VINF_SUCCESS;
1231}
1232
1233
1234/**
1235 * Adds a new (or updates the value of an existing) guest/host MSR
1236 * pair to be swapped during the world-switch as part of the
1237 * auto-load/store MSR area in the VMCS.
1238 *
1239 * @returns true if the MSR was added -and- its value was updated, false
1240 * otherwise.
1241 * @param pVCpu Pointer to the VMCPU.
1242 * @param uMsr The MSR.
1243 * @param uGuestMsr Value of the guest MSR.
1244 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1245 * necessary.
1246 */
1247static bool hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr)
1248{
1249 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1250 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1251 uint32_t i;
1252 for (i = 0; i < cMsrs; i++)
1253 {
1254 if (pGuestMsr->u32Msr == uMsr)
1255 break;
1256 pGuestMsr++;
1257 }
1258
1259 bool fAdded = false;
1260 if (i == cMsrs)
1261 {
1262 ++cMsrs;
1263 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1264 AssertRC(rc);
1265
1266 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1267 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1268 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1269
1270 fAdded = true;
1271 }
1272
1273 /* Update the MSR values in the auto-load/store MSR area. */
1274 pGuestMsr->u32Msr = uMsr;
1275 pGuestMsr->u64Value = uGuestMsrValue;
1276
1277 /* Create/update the MSR slot in the host MSR area. */
1278 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1279 pHostMsr += i;
1280 pHostMsr->u32Msr = uMsr;
1281
1282 /*
1283 * Update the host MSR only when requested by the caller AND when we're
1284 * adding it to the auto-load/store area. Otherwise, it would have been
1285 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1286 */
1287 bool fUpdatedMsrValue = false;
1288 if ( fAdded
1289 && fUpdateHostMsr)
1290 {
1291 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1292 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1293 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1294 fUpdatedMsrValue = true;
1295 }
1296
1297 return fUpdatedMsrValue;
1298}
1299
1300
1301/**
1302 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1303 * auto-load/store MSR area in the VMCS.
1304 *
1305 * Does not fail if the MSR in @a uMsr is not found in the auto-load/store MSR
1306 * area.
1307 *
1308 * @returns VBox status code.
1309 * @param pVCpu Pointer to the VMCPU.
1310 * @param uMsr The MSR.
1311 */
1312static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1313{
1314 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1315 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1316 for (uint32_t i = 0; i < cMsrs; i++)
1317 {
1318 /* Find the MSR. */
1319 if (pGuestMsr->u32Msr == uMsr)
1320 {
1321 /* If it's the last MSR, simply reduce the count. */
1322 if (i == cMsrs - 1)
1323 {
1324 --cMsrs;
1325 break;
1326 }
1327
1328 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1329 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1330 pLastGuestMsr += cMsrs;
1331 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1332 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1333
1334 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1335 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1336 pLastHostMsr += cMsrs;
1337 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1338 pHostMsr->u64Value = pLastHostMsr->u64Value;
1339 --cMsrs;
1340 break;
1341 }
1342 pGuestMsr++;
1343 }
1344
1345 /* Update the VMCS if the count changed (meaning the MSR was found). */
1346 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1347 {
1348 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1349 AssertRCReturn(rc, rc);
1350
1351 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1352 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1353 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1354 }
1355
1356 return VINF_SUCCESS;
1357}
1358
1359
1360/**
1361 * Checks if the specified guest MSR is part of the auto-load/store area in
1362 * the VMCS.
1363 *
1364 * @returns true if found, false otherwise.
1365 * @param pVCpu Pointer to the VMCPU.
1366 * @param uMsr The MSR to find.
1367 */
1368static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1369{
1370 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1371 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1372
1373 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1374 {
1375 if (pGuestMsr->u32Msr == uMsr)
1376 return true;
1377 }
1378 return false;
1379}
1380
1381
1382/**
1383 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1384 *
1385 * @param pVCpu Pointer to the VMCPU.
1386 *
1387 * @remarks No-long-jump zone!!!
1388 */
1389static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1390{
1391 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1392 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1393 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1394 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1395
1396 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1397 {
1398 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1399 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1400 }
1401
1402 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1403}
1404
1405
1406#if HC_ARCH_BITS == 64
1407/**
1408 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1409 * perform lazy restoration of the host MSRs while leaving VT-x.
1410 *
1411 * @param pVCpu Pointer to the VMCPU.
1412 *
1413 * @remarks No-long-jump zone!!!
1414 */
1415static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1416{
1417 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1418
1419 /*
1420 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1421 */
1422 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1423 {
1424 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1425 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1426 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1427 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1428 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_SAVED_HOST;
1429 }
1430}
1431
1432
1433/**
1434 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1435 * lazily while leaving VT-x.
1436 *
1437 * @returns true if it does, false otherwise.
1438 * @param pVCpu Pointer to the VMCPU.
1439 * @param uMsr The MSR to check.
1440 */
1441static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1442{
1443 NOREF(pVCpu);
1444 switch (uMsr)
1445 {
1446 case MSR_K8_LSTAR:
1447 case MSR_K6_STAR:
1448 case MSR_K8_SF_MASK:
1449 case MSR_K8_KERNEL_GS_BASE:
1450 return true;
1451 }
1452 return false;
1453}
1454
1455
1456/**
1457 * Saves a set of guests MSRs back into the guest-CPU context.
1458 *
1459 * @param pVCpu Pointer to the VMCPU.
1460 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1461 * out-of-sync. Make sure to update the required fields
1462 * before using them.
1463 *
1464 * @remarks No-long-jump zone!!!
1465 */
1466static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1467{
1468 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1469 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1470
1471 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1472 {
1473 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1474 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1475 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1476 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1477 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1478 }
1479}
1480
1481
1482/**
1483 * Loads a set of guests MSRs to allow read/passthru to the guest.
1484 *
1485 * The name of this function is slightly confusing. This function does NOT
1486 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1487 * common prefix for functions dealing with "lazy restoration" of the shared
1488 * MSRs.
1489 *
1490 * @param pVCpu Pointer to the VMCPU.
1491 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1492 * out-of-sync. Make sure to update the required fields
1493 * before using them.
1494 *
1495 * @remarks No-long-jump zone!!!
1496 */
1497static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1498{
1499 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1500 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1501
1502 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1503 if (!(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST))
1504 {
1505#define VMXLOCAL_LAZY_LOAD_GUEST_MSR(uMsr, a_GuestMsr, a_HostMsr) \
1506 do { \
1507 if (pMixedCtx->msr##a_GuestMsr != pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr) \
1508 ASMWrMsr(uMsr, pMixedCtx->msr##a_GuestMsr); \
1509 else \
1510 Assert(ASMRdMsr(uMsr) == pVCpu->hm.s.vmx.u64Host##a_HostMsr##Msr); \
1511 } while (0)
1512
1513 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_LSTAR, LSTAR, LStar);
1514 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K6_STAR, STAR, Star);
1515 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_SF_MASK, SFMASK, SFMask);
1516 VMXLOCAL_LAZY_LOAD_GUEST_MSR(MSR_K8_KERNEL_GS_BASE, KERNELGSBASE, KernelGSBase);
1517#undef VMXLOCAL_LAZY_LOAD_GUEST_MSR
1518 }
1519 else
1520 {
1521 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1522 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1523 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1524 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1525 }
1526 pVCpu->hm.s.vmx.fRestoreHostMsrs |= VMX_RESTORE_HOST_MSR_LOADED_GUEST;
1527}
1528
1529
1530/**
1531 * Performs lazy restoration of the set of host MSRs if they were previously
1532 * loaded with guest MSR values.
1533 *
1534 * @param pVCpu Pointer to the VMCPU.
1535 *
1536 * @remarks No-long-jump zone!!!
1537 * @remarks The guest MSRs should have been saved back into the guest-CPU
1538 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1539 */
1540static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1541{
1542 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1543 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1544
1545 if (pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_LOADED_GUEST)
1546 {
1547 Assert(pVCpu->hm.s.vmx.fRestoreHostMsrs & VMX_RESTORE_HOST_MSR_SAVED_HOST);
1548 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1549 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1550 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1551 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1552 }
1553 pVCpu->hm.s.vmx.fRestoreHostMsrs &= ~(VMX_RESTORE_HOST_MSR_LOADED_GUEST | VMX_RESTORE_HOST_MSR_SAVED_HOST);
1554}
1555#endif /* HC_ARCH_BITS == 64 */
1556
1557
1558/**
1559 * Verifies that our cached values of the VMCS controls are all
1560 * consistent with what's actually present in the VMCS.
1561 *
1562 * @returns VBox status code.
1563 * @param pVCpu Pointer to the VMCPU.
1564 */
1565static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1566{
1567 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1568
1569 uint32_t u32Val;
1570 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1571 AssertRCReturn(rc, rc);
1572 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1573 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1574
1575 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1576 AssertRCReturn(rc, rc);
1577 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1578 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1579
1580 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1581 AssertRCReturn(rc, rc);
1582 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1583 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1584
1585 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1586 AssertRCReturn(rc, rc);
1587 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1588 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1589
1590 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1591 AssertRCReturn(rc, rc);
1592 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1593 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1594
1595 return VINF_SUCCESS;
1596}
1597
1598
1599#ifdef VBOX_STRICT
1600/**
1601 * Verifies that our cached host EFER value has not changed
1602 * since we cached it.
1603 *
1604 * @param pVCpu Pointer to the VMCPU.
1605 */
1606static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1607{
1608 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1609
1610 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1611 {
1612 uint64_t u64Val;
1613 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, &u64Val);
1614 AssertRC(rc);
1615
1616 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1617 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1618 }
1619}
1620
1621
1622/**
1623 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1624 * VMCS are correct.
1625 *
1626 * @param pVCpu Pointer to the VMCPU.
1627 */
1628static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1629{
1630 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1631
1632 /* Verify MSR counts in the VMCS are what we think it should be. */
1633 uint32_t cMsrs;
1634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1635 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1636
1637 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1638 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1641 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1642
1643 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1644 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1645 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1646 {
1647 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1648 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32\n", pHostMsr->u32Msr,
1649 pGuestMsr->u32Msr));
1650
1651 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1652 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64\n", pHostMsr->u32Msr,
1653 pHostMsr->u64Value, u64Msr));
1654
1655 /* Verify that the permissions are as expected in the MSR bitmap. */
1656 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1657 {
1658 VMXMSREXITREAD enmRead;
1659 VMXMSREXITWRITE enmWrite;
1660 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1661 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1662 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 No passthru read permission!\n",
1663 pGuestMsr->u32Msr));
1664 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 No passthru write permission!\n",
1665 pGuestMsr->u32Msr));
1666 }
1667 }
1668}
1669# endif /* VBOX_STRICT */
1670
1671
1672/**
1673 * Flushes the TLB using EPT.
1674 *
1675 * @returns VBox status code.
1676 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1677 * enmFlush).
1678 * @param enmFlush Type of flush.
1679 *
1680 * @remarks Caller is responsible for making sure this function is called only
1681 * when NestedPaging is supported and providing @a enmFlush that is
1682 * supported by the CPU.
1683 * @remarks Can be called with interrupts disabled.
1684 */
1685static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMX_FLUSH_EPT enmFlush)
1686{
1687 uint64_t au64Descriptor[2];
1688 if (enmFlush == VMX_FLUSH_EPT_ALL_CONTEXTS)
1689 au64Descriptor[0] = 0;
1690 else
1691 {
1692 Assert(pVCpu);
1693 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1694 }
1695 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1696
1697 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1698 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1699 rc));
1700 if ( RT_SUCCESS(rc)
1701 && pVCpu)
1702 {
1703 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1704 }
1705}
1706
1707
1708/**
1709 * Flushes the TLB using VPID.
1710 *
1711 * @returns VBox status code.
1712 * @param pVM Pointer to the VM.
1713 * @param pVCpu Pointer to the VMCPU (can be NULL depending on @a
1714 * enmFlush).
1715 * @param enmFlush Type of flush.
1716 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1717 * on @a enmFlush).
1718 *
1719 * @remarks Can be called with interrupts disabled.
1720 */
1721static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMX_FLUSH_VPID enmFlush, RTGCPTR GCPtr)
1722{
1723 NOREF(pVM);
1724 AssertPtr(pVM);
1725 Assert(pVM->hm.s.vmx.fVpid);
1726
1727 uint64_t au64Descriptor[2];
1728 if (enmFlush == VMX_FLUSH_VPID_ALL_CONTEXTS)
1729 {
1730 au64Descriptor[0] = 0;
1731 au64Descriptor[1] = 0;
1732 }
1733 else
1734 {
1735 AssertPtr(pVCpu);
1736 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1737 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1738 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1739 au64Descriptor[1] = GCPtr;
1740 }
1741
1742 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1743 AssertMsg(rc == VINF_SUCCESS,
1744 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1745 if ( RT_SUCCESS(rc)
1746 && pVCpu)
1747 {
1748 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1749 }
1750}
1751
1752
1753/**
1754 * Invalidates a guest page by guest virtual address. Only relevant for
1755 * EPT/VPID, otherwise there is nothing really to invalidate.
1756 *
1757 * @returns VBox status code.
1758 * @param pVM Pointer to the VM.
1759 * @param pVCpu Pointer to the VMCPU.
1760 * @param GCVirt Guest virtual address of the page to invalidate.
1761 */
1762VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1763{
1764 AssertPtr(pVM);
1765 AssertPtr(pVCpu);
1766 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1767
1768 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1769 if (!fFlushPending)
1770 {
1771 /*
1772 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1773 * See @bugref{6043} and @bugref{6177}.
1774 *
1775 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1776 * function maybe called in a loop with individual addresses.
1777 */
1778 if (pVM->hm.s.vmx.fVpid)
1779 {
1780 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1781 {
1782 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, GCVirt);
1783 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1784 }
1785 else
1786 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1787 }
1788 else if (pVM->hm.s.fNestedPaging)
1789 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1790 }
1791
1792 return VINF_SUCCESS;
1793}
1794
1795
1796/**
1797 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1798 * otherwise there is nothing really to invalidate.
1799 *
1800 * @returns VBox status code.
1801 * @param pVM Pointer to the VM.
1802 * @param pVCpu Pointer to the VMCPU.
1803 * @param GCPhys Guest physical address of the page to invalidate.
1804 */
1805VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1806{
1807 NOREF(pVM); NOREF(GCPhys);
1808 LogFlowFunc(("%RGp\n", GCPhys));
1809
1810 /*
1811 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1812 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1813 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1814 */
1815 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1816 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1817 return VINF_SUCCESS;
1818}
1819
1820
1821/**
1822 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1823 * case where neither EPT nor VPID is supported by the CPU.
1824 *
1825 * @param pVM Pointer to the VM.
1826 * @param pVCpu Pointer to the VMCPU.
1827 * @param pCpu Pointer to the global HM struct.
1828 *
1829 * @remarks Called with interrupts disabled.
1830 */
1831static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1832{
1833 AssertPtr(pVCpu);
1834 AssertPtr(pCpu);
1835 NOREF(pVM);
1836
1837 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1838
1839 /** @todo TLB shootdown is currently not used. See hmQueueInvlPage(). */
1840#if 0
1841 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1842 pVCpu->hm.s.TlbShootdown.cPages = 0;
1843#endif
1844
1845 Assert(pCpu->idCpu != NIL_RTCPUID);
1846 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1847 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1848 pVCpu->hm.s.fForceTLBFlush = false;
1849 return;
1850}
1851
1852
1853/**
1854 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1855 *
1856 * @param pVM Pointer to the VM.
1857 * @param pVCpu Pointer to the VMCPU.
1858 * @param pCpu Pointer to the global HM CPU struct.
1859 * @remarks All references to "ASID" in this function pertains to "VPID" in
1860 * Intel's nomenclature. The reason is, to avoid confusion in compare
1861 * statements since the host-CPU copies are named "ASID".
1862 *
1863 * @remarks Called with interrupts disabled.
1864 */
1865static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1866{
1867#ifdef VBOX_WITH_STATISTICS
1868 bool fTlbFlushed = false;
1869# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1870# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1871 if (!fTlbFlushed) \
1872 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1873 } while (0)
1874#else
1875# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1876# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1877#endif
1878
1879 AssertPtr(pVM);
1880 AssertPtr(pCpu);
1881 AssertPtr(pVCpu);
1882 Assert(pCpu->idCpu != NIL_RTCPUID);
1883
1884 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1885 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1886 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1887
1888 /*
1889 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
1890 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
1891 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
1892 */
1893 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1894 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1895 {
1896 ++pCpu->uCurrentAsid;
1897 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1898 {
1899 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1900 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1901 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1902 }
1903
1904 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1905 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1906 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1907
1908 /*
1909 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1910 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1911 */
1912 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1913 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1914 HMVMX_SET_TAGGED_TLB_FLUSHED();
1915 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1916 }
1917
1918 /* Check for explicit TLB shootdowns. */
1919 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1920 {
1921 /*
1922 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
1923 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
1924 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
1925 * but not guest-physical mappings.
1926 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
1927 */
1928 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1929 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1930 HMVMX_SET_TAGGED_TLB_FLUSHED();
1931 }
1932
1933 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
1934 * where it is commented out. Support individual entry flushing
1935 * someday. */
1936#if 0
1937 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
1938 {
1939 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
1940
1941 /*
1942 * Flush individual guest entries using VPID from the TLB or as little as possible with EPT
1943 * as supported by the CPU.
1944 */
1945 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
1946 {
1947 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
1948 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
1949 }
1950 else
1951 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1952
1953 HMVMX_SET_TAGGED_TLB_FLUSHED();
1954 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
1955 pVCpu->hm.s.TlbShootdown.cPages = 0;
1956 }
1957#endif
1958
1959 pVCpu->hm.s.fForceTLBFlush = false;
1960
1961 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1962
1963 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1964 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1965 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1966 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1967 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1968 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1969 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1970 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1971 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1972
1973 /* Update VMCS with the VPID. */
1974 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
1975 AssertRC(rc);
1976
1977#undef HMVMX_SET_TAGGED_TLB_FLUSHED
1978}
1979
1980
1981/**
1982 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
1983 *
1984 * @returns VBox status code.
1985 * @param pVM Pointer to the VM.
1986 * @param pVCpu Pointer to the VMCPU.
1987 * @param pCpu Pointer to the global HM CPU struct.
1988 *
1989 * @remarks Called with interrupts disabled.
1990 */
1991static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1992{
1993 AssertPtr(pVM);
1994 AssertPtr(pVCpu);
1995 AssertPtr(pCpu);
1996 Assert(pCpu->idCpu != NIL_RTCPUID);
1997 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
1998 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
1999
2000 /*
2001 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2002 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2003 */
2004 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2005 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2006 {
2007 pVCpu->hm.s.fForceTLBFlush = true;
2008 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2009 }
2010
2011 /* Check for explicit TLB shootdown flushes. */
2012 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2013 {
2014 pVCpu->hm.s.fForceTLBFlush = true;
2015 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2016 }
2017
2018 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2019 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2020
2021 if (pVCpu->hm.s.fForceTLBFlush)
2022 {
2023 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2024 pVCpu->hm.s.fForceTLBFlush = false;
2025 }
2026 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2027 * where it is commented out. Support individual entry flushing
2028 * someday. */
2029#if 0
2030 else
2031 {
2032 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2033 {
2034 /* We cannot flush individual entries without VPID support. Flush using EPT. */
2035 STAM_COUNTER_INC(&pVCpu->hm.s.StatTlbShootdown);
2036 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2037 }
2038 else
2039 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2040
2041 pVCpu->hm.s.TlbShootdown.cPages = 0;
2042 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2043 }
2044#endif
2045}
2046
2047
2048/**
2049 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2050 *
2051 * @returns VBox status code.
2052 * @param pVM Pointer to the VM.
2053 * @param pVCpu Pointer to the VMCPU.
2054 * @param pCpu Pointer to the global HM CPU struct.
2055 *
2056 * @remarks Called with interrupts disabled.
2057 */
2058static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2059{
2060 AssertPtr(pVM);
2061 AssertPtr(pVCpu);
2062 AssertPtr(pCpu);
2063 Assert(pCpu->idCpu != NIL_RTCPUID);
2064 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2065 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2066
2067 /*
2068 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2069 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2070 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2071 */
2072 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2073 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2074 {
2075 pVCpu->hm.s.fForceTLBFlush = true;
2076 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2077 }
2078
2079 /* Check for explicit TLB shootdown flushes. */
2080 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2081 {
2082 /*
2083 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2084 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2085 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2086 */
2087 pVCpu->hm.s.fForceTLBFlush = true;
2088 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2089 }
2090
2091 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2092 if (pVCpu->hm.s.fForceTLBFlush)
2093 {
2094 ++pCpu->uCurrentAsid;
2095 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2096 {
2097 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2098 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2099 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2100 }
2101
2102 pVCpu->hm.s.fForceTLBFlush = false;
2103 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2104 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2105 if (pCpu->fFlushAsidBeforeUse)
2106 {
2107 if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_SINGLE_CONTEXT)
2108 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2109 else if (pVM->hm.s.vmx.enmFlushVpid == VMX_FLUSH_VPID_ALL_CONTEXTS)
2110 {
2111 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_ALL_CONTEXTS, 0 /* GCPtr */);
2112 pCpu->fFlushAsidBeforeUse = false;
2113 }
2114 else
2115 {
2116 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2117 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2118 }
2119 }
2120 }
2121 /** @todo We never set VMCPU_FF_TLB_SHOOTDOWN anywhere. See hmQueueInvlPage()
2122 * where it is commented out. Support individual entry flushing
2123 * someday. */
2124#if 0
2125 else
2126 {
2127 AssertMsg(pVCpu->hm.s.uCurrentAsid && pCpu->uCurrentAsid,
2128 ("hm->uCurrentAsid=%lu hm->cTlbFlushes=%lu cpu->uCurrentAsid=%lu cpu->cTlbFlushes=%lu\n",
2129 pVCpu->hm.s.uCurrentAsid, pVCpu->hm.s.cTlbFlushes,
2130 pCpu->uCurrentAsid, pCpu->cTlbFlushes));
2131
2132 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN))
2133 {
2134 /* Flush individual guest entries using VPID or as little as possible with EPT as supported by the CPU. */
2135 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2136 {
2137 for (uint32_t i = 0; i < pVCpu->hm.s.TlbShootdown.cPages; i++)
2138 hmR0VmxFlushVpid(pVM, pVCpu, VMX_FLUSH_VPID_INDIV_ADDR, pVCpu->hm.s.TlbShootdown.aPages[i]);
2139 }
2140 else
2141 hmR0VmxFlushVpid(pVM, pVCpu, pVM->hm.s.vmx.enmFlushVpid, 0 /* GCPtr */);
2142
2143 pVCpu->hm.s.TlbShootdown.cPages = 0;
2144 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_SHOOTDOWN);
2145 }
2146 else
2147 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch);
2148 }
2149#endif
2150
2151 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2152 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2153 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2154 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2155 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2156 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2157 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2158
2159 int rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, pVCpu->hm.s.uCurrentAsid);
2160 AssertRC(rc);
2161}
2162
2163
2164/**
2165 * Flushes the guest TLB entry based on CPU capabilities.
2166 *
2167 * @param pVCpu Pointer to the VMCPU.
2168 * @param pCpu Pointer to the global HM CPU struct.
2169 */
2170DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2171{
2172#ifdef HMVMX_ALWAYS_FLUSH_TLB
2173 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2174#endif
2175 PVM pVM = pVCpu->CTX_SUFF(pVM);
2176 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2177 {
2178 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2179 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2180 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2181 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2182 default:
2183 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2184 break;
2185 }
2186
2187 /* VMCPU_FF_TLB_SHOOTDOWN is unused. */
2188 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_SHOOTDOWN));
2189
2190 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2191}
2192
2193
2194/**
2195 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2196 * TLB entries from the host TLB before VM-entry.
2197 *
2198 * @returns VBox status code.
2199 * @param pVM Pointer to the VM.
2200 */
2201static int hmR0VmxSetupTaggedTlb(PVM pVM)
2202{
2203 /*
2204 * Determine optimal flush type for Nested Paging.
2205 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2206 * guest execution (see hmR3InitFinalizeR0()).
2207 */
2208 if (pVM->hm.s.fNestedPaging)
2209 {
2210 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2211 {
2212 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2213 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_SINGLE_CONTEXT;
2214 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2215 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_ALL_CONTEXTS;
2216 else
2217 {
2218 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2219 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221 }
2222
2223 /* Make sure the write-back cacheable memory type for EPT is supported. */
2224 if (!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB))
2225 {
2226 LogRel(("hmR0VmxSetupTaggedTlb: Unsupported EPTP memory type %#x.\n", pVM->hm.s.vmx.Msrs.u64EptVpidCaps));
2227 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229 }
2230 }
2231 else
2232 {
2233 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2234 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NOT_SUPPORTED;
2235 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2236 }
2237 }
2238
2239 /*
2240 * Determine optimal flush type for VPID.
2241 */
2242 if (pVM->hm.s.vmx.fVpid)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2245 {
2246 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2247 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_SINGLE_CONTEXT;
2248 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2249 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_ALL_CONTEXTS;
2250 else
2251 {
2252 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2253 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2254 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2255 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2256 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2257 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2258 pVM->hm.s.vmx.fVpid = false;
2259 }
2260 }
2261 else
2262 {
2263 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2264 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2265 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NOT_SUPPORTED;
2266 pVM->hm.s.vmx.fVpid = false;
2267 }
2268 }
2269
2270 /*
2271 * Setup the handler for flushing tagged-TLBs.
2272 */
2273 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2274 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2275 else if (pVM->hm.s.fNestedPaging)
2276 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2277 else if (pVM->hm.s.vmx.fVpid)
2278 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2279 else
2280 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2281 return VINF_SUCCESS;
2282}
2283
2284
2285/**
2286 * Sets up pin-based VM-execution controls in the VMCS.
2287 *
2288 * @returns VBox status code.
2289 * @param pVM Pointer to the VM.
2290 * @param pVCpu Pointer to the VMCPU.
2291 */
2292static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2293{
2294 AssertPtr(pVM);
2295 AssertPtr(pVCpu);
2296
2297 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2298 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2299
2300 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts causes a VM-exits. */
2301 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts causes a VM-exit. */
2302 Assert(!(val & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI));
2303
2304 /* Enable the VMX preemption timer. */
2305 if (pVM->hm.s.vmx.fUsePreemptTimer)
2306 {
2307 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2308 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2309 }
2310
2311 if ((val & zap) != val)
2312 {
2313 LogRel(("hmR0VmxSetupPinCtls: invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2314 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2315 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2316 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2317 }
2318
2319 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2320 AssertRCReturn(rc, rc);
2321
2322 /* Update VCPU with the currently set pin-based VM-execution controls. */
2323 pVCpu->hm.s.vmx.u32PinCtls = val;
2324 return rc;
2325}
2326
2327
2328/**
2329 * Sets up processor-based VM-execution controls in the VMCS.
2330 *
2331 * @returns VBox status code.
2332 * @param pVM Pointer to the VM.
2333 * @param pVMCPU Pointer to the VMCPU.
2334 */
2335static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2336{
2337 AssertPtr(pVM);
2338 AssertPtr(pVCpu);
2339
2340 int rc = VERR_INTERNAL_ERROR_5;
2341 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2342 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2343
2344 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2345 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2346 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2347 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2348 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2349 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2350 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2351
2352 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2353 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2354 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2355 {
2356 LogRel(("hmR0VmxSetupProcCtls: unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2357 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2358 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2359 }
2360
2361 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2362 if (!pVM->hm.s.fNestedPaging)
2363 {
2364 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2365 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2366 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2367 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2368 }
2369
2370 /* Use TPR shadowing if supported by the CPU. */
2371 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2372 {
2373 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2374 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2375 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2376 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2377 AssertRCReturn(rc, rc);
2378
2379 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2380 /* CR8 writes causes a VM-exit based on TPR threshold. */
2381 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2382 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2383 }
2384 else
2385 {
2386 /*
2387 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2388 * Set this control only for 64-bit guests.
2389 */
2390 if (pVM->hm.s.fAllow64BitGuests)
2391 {
2392 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads causes a VM-exit. */
2393 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes causes a VM-exit. */
2394 }
2395 }
2396
2397 /* Use MSR-bitmaps if supported by the CPU. */
2398 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2399 {
2400 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2401
2402 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2403 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2404 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2405 AssertRCReturn(rc, rc);
2406
2407 /*
2408 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2409 * automatically as dedicated fields in the VMCS.
2410 */
2411 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2412 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2413 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2414 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2415 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2416
2417#if HC_ARCH_BITS == 64
2418 /*
2419 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2420 */
2421 if (pVM->hm.s.fAllow64BitGuests)
2422 {
2423 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2424 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2425 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2426 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2427 }
2428#endif
2429 }
2430
2431 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2432 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2433 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2434
2435 if ((val & zap) != val)
2436 {
2437 LogRel(("hmR0VmxSetupProcCtls: invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2438 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2439 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2440 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2441 }
2442
2443 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2444 AssertRCReturn(rc, rc);
2445
2446 /* Update VCPU with the currently set processor-based VM-execution controls. */
2447 pVCpu->hm.s.vmx.u32ProcCtls = val;
2448
2449 /*
2450 * Secondary processor-based VM-execution controls.
2451 */
2452 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2453 {
2454 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2455 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2456
2457 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2458 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2459
2460 if (pVM->hm.s.fNestedPaging)
2461 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2462 else
2463 {
2464 /*
2465 * Without Nested Paging, INVPCID should cause a VM-exit. Enabling this bit causes the CPU to refer to
2466 * VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT when INVPCID is executed by the guest.
2467 * See Intel spec. 25.4 "Changes to instruction behaviour in VMX non-root operation".
2468 */
2469 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2470 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2471 }
2472
2473 if (pVM->hm.s.vmx.fVpid)
2474 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2475
2476 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2477 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2478
2479 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2480 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2481 * done dynamically. */
2482 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2483 {
2484 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2485 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2486 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2487 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2488 AssertRCReturn(rc, rc);
2489 }
2490
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2492 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2493
2494 if ((val & zap) != val)
2495 {
2496 LogRel(("hmR0VmxSetupProcCtls: invalid secondary processor-based VM-execution controls combo! "
2497 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2498 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2499 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2500 }
2501
2502 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2503 AssertRCReturn(rc, rc);
2504
2505 /* Update VCPU with the currently set secondary processor-based VM-execution controls. */
2506 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2507 }
2508 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2509 {
2510 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2511 "available\n"));
2512 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2513 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2514 }
2515
2516 return VINF_SUCCESS;
2517}
2518
2519
2520/**
2521 * Sets up miscellaneous (everything other than Pin & Processor-based
2522 * VM-execution) control fields in the VMCS.
2523 *
2524 * @returns VBox status code.
2525 * @param pVM Pointer to the VM.
2526 * @param pVCpu Pointer to the VMCPU.
2527 */
2528static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2529{
2530 NOREF(pVM);
2531 AssertPtr(pVM);
2532 AssertPtr(pVCpu);
2533
2534 int rc = VERR_GENERAL_FAILURE;
2535
2536 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2537#if 0
2538 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2539 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0); AssertRCReturn(rc, rc);
2540 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0); AssertRCReturn(rc, rc);
2541
2542 /*
2543 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2544 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2545 * We thus use the exception bitmap to control it rather than use both.
2546 */
2547 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0); AssertRCReturn(rc, rc);
2548 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0); AssertRCReturn(rc, rc);
2549
2550 /** @todo Explore possibility of using IO-bitmaps. */
2551 /* All IO & IOIO instructions cause VM-exits. */
2552 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0); AssertRCReturn(rc, rc);
2553 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0); AssertRCReturn(rc, rc);
2554
2555 /* Initialize the MSR-bitmap area. */
2556 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2557 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0); AssertRCReturn(rc, rc);
2558 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0); AssertRCReturn(rc, rc);
2559#endif
2560
2561 /* Setup MSR auto-load/store area. */
2562 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2563 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2564 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2565 AssertRCReturn(rc, rc);
2566 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2567 AssertRCReturn(rc, rc);
2568
2569 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2570 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2571 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2572 AssertRCReturn(rc, rc);
2573
2574 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2575 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2576 AssertRCReturn(rc, rc);
2577
2578 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2579#if 0
2580 /* Setup debug controls */
2581 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2582 AssertRCReturn(rc, rc);
2583 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2584 AssertRCReturn(rc, rc);
2585#endif
2586
2587 return rc;
2588}
2589
2590
2591/**
2592 * Sets up the initial exception bitmap in the VMCS based on static conditions
2593 * (i.e. conditions that cannot ever change after starting the VM).
2594 *
2595 * @returns VBox status code.
2596 * @param pVM Pointer to the VM.
2597 * @param pVCpu Pointer to the VMCPU.
2598 */
2599static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2600{
2601 AssertPtr(pVM);
2602 AssertPtr(pVCpu);
2603
2604 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2605
2606 uint32_t u32XcptBitmap = 0;
2607
2608 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2609 if (!pVM->hm.s.fNestedPaging)
2610 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2611
2612 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2613 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2614 AssertRCReturn(rc, rc);
2615 return rc;
2616}
2617
2618
2619/**
2620 * Sets up the initial guest-state mask. The guest-state mask is consulted
2621 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2622 * for the nested virtualization case (as it would cause a VM-exit).
2623 *
2624 * @param pVCpu Pointer to the VMCPU.
2625 */
2626static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2627{
2628 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2629 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2630 return VINF_SUCCESS;
2631}
2632
2633
2634/**
2635 * Does per-VM VT-x initialization.
2636 *
2637 * @returns VBox status code.
2638 * @param pVM Pointer to the VM.
2639 */
2640VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2641{
2642 LogFlowFunc(("pVM=%p\n", pVM));
2643
2644 int rc = hmR0VmxStructsAlloc(pVM);
2645 if (RT_FAILURE(rc))
2646 {
2647 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2648 return rc;
2649 }
2650
2651 return VINF_SUCCESS;
2652}
2653
2654
2655/**
2656 * Does per-VM VT-x termination.
2657 *
2658 * @returns VBox status code.
2659 * @param pVM Pointer to the VM.
2660 */
2661VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2662{
2663 LogFlowFunc(("pVM=%p\n", pVM));
2664
2665#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2666 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2667 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2668#endif
2669 hmR0VmxStructsFree(pVM);
2670 return VINF_SUCCESS;
2671}
2672
2673
2674/**
2675 * Sets up the VM for execution under VT-x.
2676 * This function is only called once per-VM during initialization.
2677 *
2678 * @returns VBox status code.
2679 * @param pVM Pointer to the VM.
2680 */
2681VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2682{
2683 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2684 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2685
2686 LogFlowFunc(("pVM=%p\n", pVM));
2687
2688 /*
2689 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2690 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0().
2691 */
2692 /* -XXX- change hmR3InitFinalizeR0Intel() to fail if pRealModeTSS alloc fails. */
2693 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2694 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2695 || !pVM->hm.s.vmx.pRealModeTSS))
2696 {
2697 LogRel(("VMXR0SetupVM: invalid real-on-v86 state.\n"));
2698 return VERR_INTERNAL_ERROR;
2699 }
2700
2701#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2702 /*
2703 * This is for the darwin 32-bit/PAE kernels trying to execute 64-bit guests. We don't bother with
2704 * the 32<->64 switcher in this case. This is a rare, legacy use-case with barely any test coverage.
2705 */
2706 if ( pVM->hm.s.fAllow64BitGuests
2707 && !HMVMX_IS_64BIT_HOST_MODE())
2708 {
2709 LogRel(("VMXR0SetupVM: Unsupported guest and host paging mode combination.\n"));
2710 return VERR_PGM_UNSUPPORTED_HOST_PAGING_MODE;
2711 }
2712#endif
2713
2714 /* Initialize these always, see hmR3InitFinalizeR0().*/
2715 pVM->hm.s.vmx.enmFlushEpt = VMX_FLUSH_EPT_NONE;
2716 pVM->hm.s.vmx.enmFlushVpid = VMX_FLUSH_VPID_NONE;
2717
2718 /* Setup the tagged-TLB flush handlers. */
2719 int rc = hmR0VmxSetupTaggedTlb(pVM);
2720 if (RT_FAILURE(rc))
2721 {
2722 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2723 return rc;
2724 }
2725
2726 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2727 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2728#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2729 if ( HMVMX_IS_64BIT_HOST_MODE()
2730 && (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2731 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2732 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2733 {
2734 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2735 }
2736#endif
2737
2738 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2739 {
2740 PVMCPU pVCpu = &pVM->aCpus[i];
2741 AssertPtr(pVCpu);
2742 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2743
2744 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2745 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2746
2747 /* Set revision dword at the beginning of the VMCS structure. */
2748 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2749
2750 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2751 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2752 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2753 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2754
2755 /* Load this VMCS as the current VMCS. */
2756 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2757 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2758 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2759
2760 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2761 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2762 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2763
2764 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2765 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2766 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2767
2768 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2769 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2770 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2771
2772 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2773 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2774 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2775
2776 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2777 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2778 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2779
2780#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
2781 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2783 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2784#endif
2785
2786 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2787 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2788 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2789 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2790
2791 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2792
2793 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2794 }
2795
2796 return VINF_SUCCESS;
2797}
2798
2799
2800/**
2801 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2802 * the VMCS.
2803 *
2804 * @returns VBox status code.
2805 * @param pVM Pointer to the VM.
2806 * @param pVCpu Pointer to the VMCPU.
2807 */
2808DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2809{
2810 NOREF(pVM); NOREF(pVCpu);
2811
2812 RTCCUINTREG uReg = ASMGetCR0();
2813 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2814 AssertRCReturn(rc, rc);
2815
2816#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2817 /* For the darwin 32-bit hybrid kernel, we need the 64-bit CR3 as it uses 64-bit paging. */
2818 if (HMVMX_IS_64BIT_HOST_MODE())
2819 {
2820 uint64_t uRegCR3 = HMR0Get64bitCR3();
2821 rc = VMXWriteVmcs64(VMX_VMCS_HOST_CR3, uRegCR3);
2822 }
2823 else
2824#endif
2825 {
2826 uReg = ASMGetCR3();
2827 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2828 }
2829 AssertRCReturn(rc, rc);
2830
2831 uReg = ASMGetCR4();
2832 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2833 AssertRCReturn(rc, rc);
2834 return rc;
2835}
2836
2837
2838#if HC_ARCH_BITS == 64
2839/**
2840 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2841 * requirements. See hmR0VmxSaveHostSegmentRegs().
2842 */
2843# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2844 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2845 { \
2846 bool fValidSelector = true; \
2847 if ((selValue) & X86_SEL_LDT) \
2848 { \
2849 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2850 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2851 } \
2852 if (fValidSelector) \
2853 { \
2854 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2855 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2856 } \
2857 (selValue) = 0; \
2858 }
2859#endif
2860
2861
2862/**
2863 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2864 * the host-state area in the VMCS.
2865 *
2866 * @returns VBox status code.
2867 * @param pVM Pointer to the VM.
2868 * @param pVCpu Pointer to the VMCPU.
2869 */
2870DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2871{
2872 NOREF(pVM);
2873 int rc = VERR_INTERNAL_ERROR_5;
2874
2875#if HC_ARCH_BITS == 64
2876 /*
2877 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2878 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2879 */
2880 AssertMsgReturn(!(pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED),
2881 ("Re-saving host-state after executing guest code without leaving VT-x!\n"), VERR_WRONG_ORDER);
2882#endif
2883
2884 /*
2885 * Host DS, ES, FS and GS segment registers.
2886 */
2887#if HC_ARCH_BITS == 64
2888 RTSEL uSelDS = ASMGetDS();
2889 RTSEL uSelES = ASMGetES();
2890 RTSEL uSelFS = ASMGetFS();
2891 RTSEL uSelGS = ASMGetGS();
2892#else
2893 RTSEL uSelDS = 0;
2894 RTSEL uSelES = 0;
2895 RTSEL uSelFS = 0;
2896 RTSEL uSelGS = 0;
2897#endif
2898
2899 /* Recalculate which host-state bits need to be manually restored. */
2900 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2901
2902 /*
2903 * Host CS and SS segment registers.
2904 */
2905#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2906 RTSEL uSelCS;
2907 RTSEL uSelSS;
2908 if (HMVMX_IS_64BIT_HOST_MODE())
2909 {
2910 uSelCS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelCS;
2911 uSelSS = (RTSEL)(uintptr_t)&SUPR0Abs64bitKernelSS;
2912 }
2913 else
2914 {
2915 /* Seems darwin uses the LDT (TI flag is set) in the CS & SS selectors which VT-x doesn't like. */
2916 uSelCS = (RTSEL)(uintptr_t)&SUPR0AbsKernelCS;
2917 uSelSS = (RTSEL)(uintptr_t)&SUPR0AbsKernelSS;
2918 }
2919#else
2920 RTSEL uSelCS = ASMGetCS();
2921 RTSEL uSelSS = ASMGetSS();
2922#endif
2923
2924 /*
2925 * Host TR segment register.
2926 */
2927 RTSEL uSelTR = ASMGetTR();
2928
2929#if HC_ARCH_BITS == 64
2930 /*
2931 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
2932 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2933 */
2934 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2935 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2936 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2937 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2938# undef VMXLOCAL_ADJUST_HOST_SEG
2939#endif
2940
2941 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2942 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2943 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2944 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2945 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2946 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2947 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2948 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2949 Assert(uSelCS);
2950 Assert(uSelTR);
2951
2952 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2953#if 0
2954 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2955 Assert(uSelSS != 0);
2956#endif
2957
2958 /* Write these host selector fields into the host-state area in the VMCS. */
2959 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_CS, uSelCS); AssertRCReturn(rc, rc);
2960 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_SS, uSelSS); AssertRCReturn(rc, rc);
2961#if HC_ARCH_BITS == 64
2962 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_DS, uSelDS); AssertRCReturn(rc, rc);
2963 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_ES, uSelES); AssertRCReturn(rc, rc);
2964 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_FS, uSelFS); AssertRCReturn(rc, rc);
2965 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_GS, uSelGS); AssertRCReturn(rc, rc);
2966#endif
2967 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_FIELD_TR, uSelTR); AssertRCReturn(rc, rc);
2968
2969 /*
2970 * Host GDTR and IDTR.
2971 */
2972 RTGDTR Gdtr;
2973 RT_ZERO(Gdtr);
2974#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
2975 if (HMVMX_IS_64BIT_HOST_MODE())
2976 {
2977 X86XDTR64 Gdtr64;
2978 X86XDTR64 Idtr64;
2979 HMR0Get64bitGdtrAndIdtr(&Gdtr64, &Idtr64);
2980 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GDTR_BASE, Gdtr64.uAddr); AssertRCReturn(rc, rc);
2981 rc = VMXWriteVmcs64(VMX_VMCS_HOST_IDTR_BASE, Idtr64.uAddr); AssertRCReturn(rc, rc);
2982
2983 Gdtr.cbGdt = Gdtr64.cb;
2984 Gdtr.pGdt = (uintptr_t)Gdtr64.uAddr;
2985 }
2986 else
2987#endif
2988 {
2989 RTIDTR Idtr;
2990 ASMGetGDTR(&Gdtr);
2991 ASMGetIDTR(&Idtr);
2992 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt); AssertRCReturn(rc, rc);
2993 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt); AssertRCReturn(rc, rc);
2994
2995#if HC_ARCH_BITS == 64
2996 /*
2997 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
2998 * maximum limit (0xffff) on every VM-exit.
2999 */
3000 if (Gdtr.cbGdt != 0xffff)
3001 {
3002 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3003 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3004 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3005 }
3006
3007 /*
3008 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3009 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3010 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3011 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3012 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3013 * hosts where we are pretty sure it won't cause trouble.
3014 */
3015# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3016 if (Idtr.cbIdt < 0x0fff)
3017# else
3018 if (Idtr.cbIdt != 0xffff)
3019# endif
3020 {
3021 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3022 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024 }
3025#endif
3026 }
3027
3028 /*
3029 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3030 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3031 */
3032 if ((uSelTR | X86_SEL_RPL_LDT) > Gdtr.cbGdt)
3033 {
3034 AssertMsgFailed(("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt));
3035 return VERR_VMX_INVALID_HOST_STATE;
3036 }
3037
3038 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3039#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3040 if (HMVMX_IS_64BIT_HOST_MODE())
3041 {
3042 /* We need the 64-bit TR base for hybrid darwin. */
3043 uint64_t u64TRBase = X86DESC64_BASE((PX86DESC64)pDesc);
3044 rc = VMXWriteVmcs64(VMX_VMCS_HOST_TR_BASE, u64TRBase);
3045 }
3046 else
3047#endif
3048 {
3049 uintptr_t uTRBase;
3050#if HC_ARCH_BITS == 64
3051 uTRBase = X86DESC64_BASE(pDesc);
3052
3053 /*
3054 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3055 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3056 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3057 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3058 *
3059 * [1] See Intel spec. 3.5 "System Descriptor Types".
3060 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3061 */
3062 Assert(pDesc->System.u4Type == 11);
3063 if ( pDesc->System.u16LimitLow != 0x67
3064 || pDesc->System.u4LimitHigh)
3065 {
3066 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3067 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3068
3069 /* Store the GDTR here as we need it while restoring TR. */
3070 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3071 }
3072#else
3073 uTRBase = X86DESC_BASE(pDesc);
3074#endif
3075 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3076 }
3077 AssertRCReturn(rc, rc);
3078
3079 /*
3080 * Host FS base and GS base.
3081 */
3082#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3083 if (HMVMX_IS_64BIT_HOST_MODE())
3084 {
3085 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3086 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3087 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase); AssertRCReturn(rc, rc);
3088 rc = VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase); AssertRCReturn(rc, rc);
3089
3090# if HC_ARCH_BITS == 64
3091 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3092 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3093 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3096# endif
3097 }
3098#endif
3099 return rc;
3100}
3101
3102
3103/**
3104 * Saves certain host MSRs in the VM-Exit MSR-load area and some in the
3105 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3106 * the host after every successful VM-exit.
3107 *
3108 * @returns VBox status code.
3109 * @param pVM Pointer to the VM.
3110 * @param pVCpu Pointer to the VMCPU.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3115{
3116 NOREF(pVM);
3117
3118 AssertPtr(pVCpu);
3119 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3120
3121 int rc = VINF_SUCCESS;
3122#if HC_ARCH_BITS == 64
3123 if (pVM->hm.s.fAllow64BitGuests)
3124 hmR0VmxLazySaveHostMsrs(pVCpu);
3125#endif
3126
3127 /*
3128 * Host Sysenter MSRs.
3129 */
3130 rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3131 AssertRCReturn(rc, rc);
3132#ifdef VBOX_WITH_HYBRID_32BIT_KERNEL
3133 if (HMVMX_IS_64BIT_HOST_MODE())
3134 {
3135 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3136 AssertRCReturn(rc, rc);
3137 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3138 }
3139 else
3140 {
3141 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3142 AssertRCReturn(rc, rc);
3143 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3144 }
3145#elif HC_ARCH_BITS == 32
3146 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3147 AssertRCReturn(rc, rc);
3148 rc = VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3149#else
3150 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3151 AssertRCReturn(rc, rc);
3152 rc = VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3153#endif
3154 AssertRCReturn(rc, rc);
3155
3156 /*
3157 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3158 */
3159#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3160 if ( HMVMX_IS_64BIT_HOST_MODE()
3161 && pVM->hm.s.vmx.fSupportsVmcsEfer)
3162 {
3163 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_FIELD_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3164 AssertRCReturn(rc, rc);
3165 }
3166#endif
3167
3168 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3169 * hmR0VmxLoadGuestExitCtls() !! */
3170
3171 return rc;
3172}
3173
3174
3175/**
3176 * Figures out if we need to swap the EFER MSR which is
3177 * particularly expensive.
3178 *
3179 * We check all relevant bits. For now, that's everything
3180 * besides LMA/LME, as these two bits are handled by VM-entry,
3181 * see hmR0VmxLoadGuestExitCtls() and
3182 * hmR0VMxLoadGuestEntryCtls().
3183 *
3184 * @returns true if we need to load guest EFER, false otherwise.
3185 * @param pVCpu Pointer to the VMCPU.
3186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3187 * out-of-sync. Make sure to update the required fields
3188 * before using them.
3189 *
3190 * @remarks Requires EFER, CR4.
3191 * @remarks No-long-jump zone!!!
3192 */
3193static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3194{
3195 PVM pVM = pVCpu->CTX_SUFF(pVM);
3196 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3197 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3198
3199 /*
3200 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3201 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3202 */
3203 if ( pVM->hm.s.fAllow64BitGuests
3204 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3205 {
3206 return true;
3207 }
3208
3209 /*
3210 * If the guest uses PAE and EFER.NXE bit differs, we need to swap as it affects guest paging.
3211 * 64-bit paging implies CR4.PAE as well. See Intel spec. 4.5 "IA32e Paging".
3212 */
3213 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3214 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3215 {
3216 return true;
3217 }
3218
3219 /** @todo Check the latest Intel spec. for any other bits,
3220 * like SMEP/SMAP? */
3221 return false;
3222}
3223
3224
3225/**
3226 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3227 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3228 * controls".
3229 *
3230 * @returns VBox status code.
3231 * @param pVCpu Pointer to the VMCPU.
3232 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3233 * out-of-sync. Make sure to update the required fields
3234 * before using them.
3235 *
3236 * @remarks Requires EFER.
3237 * @remarks No-long-jump zone!!!
3238 */
3239DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3240{
3241 int rc = VINF_SUCCESS;
3242 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3243 {
3244 PVM pVM = pVCpu->CTX_SUFF(pVM);
3245 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3246 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3247
3248 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3249 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3250
3251 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3252 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3253 {
3254 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3255 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3256 }
3257 else
3258 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3259
3260 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3261#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3262 if ( HMVMX_IS_64BIT_HOST_MODE()
3263 && pVM->hm.s.vmx.fSupportsVmcsEfer
3264 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3265 {
3266 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3267 Log4(("Load: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3268 }
3269#endif
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((val & zap) != val)
3281 {
3282 LogRel(("hmR0VmxLoadGuestEntryCtls: invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3289 AssertRCReturn(rc, rc);
3290
3291 /* Update VCPU with the currently set VM-exit controls. */
3292 pVCpu->hm.s.vmx.u32EntryCtls = val;
3293 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3294 }
3295 return rc;
3296}
3297
3298
3299/**
3300 * Sets up the VM-exit controls in the VMCS.
3301 *
3302 * @returns VBox status code.
3303 * @param pVM Pointer to the VM.
3304 * @param pVCpu Pointer to the VMCPU.
3305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3306 * out-of-sync. Make sure to update the required fields
3307 * before using them.
3308 *
3309 * @remarks Requires EFER.
3310 */
3311DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3312{
3313 NOREF(pMixedCtx);
3314
3315 int rc = VINF_SUCCESS;
3316 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3317 {
3318 PVM pVM = pVCpu->CTX_SUFF(pVM);
3319 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3320 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3321
3322 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3323 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3324
3325 /*
3326 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3327 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3328 */
3329#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
3330 if (HMVMX_IS_64BIT_HOST_MODE())
3331 {
3332 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3333 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3334
3335 /* If the newer VMCS fields for managing EFER exists, use it. */
3336 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3337 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3338 {
3339 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3340 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3341 }
3342 }
3343 else
3344 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3345#elif HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3346 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3347 {
3348 /* The switcher returns to long mode, EFER is managed by the switcher. */
3349 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3350 Log4(("Load: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3351 }
3352 else
3353 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3354#endif
3355
3356 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3357 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3358
3359 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3360 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3361 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3362
3363 if (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER)
3364 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3365
3366 if ((val & zap) != val)
3367 {
3368 LogRel(("hmR0VmxSetupProcCtls: invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3369 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3370 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3371 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3372 }
3373
3374 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3375 AssertRCReturn(rc, rc);
3376
3377 /* Update VCPU with the currently set VM-exit controls. */
3378 pVCpu->hm.s.vmx.u32ExitCtls = val;
3379 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3380 }
3381 return rc;
3382}
3383
3384
3385/**
3386 * Loads the guest APIC and related state.
3387 *
3388 * @returns VBox status code.
3389 * @param pVM Pointer to the VM.
3390 * @param pVCpu Pointer to the VMCPU.
3391 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3392 * out-of-sync. Make sure to update the required fields
3393 * before using them.
3394 */
3395DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3396{
3397 NOREF(pMixedCtx);
3398
3399 int rc = VINF_SUCCESS;
3400 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE))
3401 {
3402 /* Setup TPR shadowing. Also setup TPR patching for 32-bit guests. */
3403 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3404 {
3405 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3406
3407 bool fPendingIntr = false;
3408 uint8_t u8Tpr = 0;
3409 uint8_t u8PendingIntr = 0;
3410 rc = PDMApicGetTPR(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3411 AssertRCReturn(rc, rc);
3412
3413 /*
3414 * If there are external interrupts pending but masked by the TPR value, instruct VT-x to cause a VM-exit when
3415 * the guest lowers its TPR below the highest-priority pending interrupt and we can deliver the interrupt.
3416 * If there are no external interrupts pending, set threshold to 0 to not cause a VM-exit. We will eventually deliver
3417 * the interrupt when we VM-exit for other reasons.
3418 */
3419 pVCpu->hm.s.vmx.pbVirtApic[0x80] = u8Tpr; /* Offset 0x80 is TPR in the APIC MMIO range. */
3420 uint32_t u32TprThreshold = 0;
3421 if (fPendingIntr)
3422 {
3423 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3424 const uint8_t u8PendingPriority = (u8PendingIntr >> 4) & 0xf;
3425 const uint8_t u8TprPriority = (u8Tpr >> 4) & 0xf;
3426 if (u8PendingPriority <= u8TprPriority)
3427 u32TprThreshold = u8PendingPriority;
3428 else
3429 u32TprThreshold = u8TprPriority; /* Required for Vista 64-bit guest, see @bugref{6398}. */
3430 }
3431 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3432
3433 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3434 AssertRCReturn(rc, rc);
3435 }
3436
3437 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
3438 }
3439 return rc;
3440}
3441
3442
3443/**
3444 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3445 *
3446 * @returns Guest's interruptibility-state.
3447 * @param pVCpu Pointer to the VMCPU.
3448 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3449 * out-of-sync. Make sure to update the required fields
3450 * before using them.
3451 *
3452 * @remarks No-long-jump zone!!!
3453 * @remarks Has side-effects with VMCPU_FF_INHIBIT_INTERRUPTS force-flag.
3454 */
3455DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3456{
3457 /*
3458 * Instructions like STI and MOV SS inhibit interrupts till the next instruction completes. Check if we should
3459 * inhibit interrupts or clear any existing interrupt-inhibition.
3460 */
3461 uint32_t uIntrState = 0;
3462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3463 {
3464 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3465 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3466 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3467 if (pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 /*
3470 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3471 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3472 */
3473 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3474 }
3475 else if (pMixedCtx->eflags.Bits.u1IF)
3476 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3477 else
3478 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3479 }
3480 return uIntrState;
3481}
3482
3483
3484/**
3485 * Loads the guest's interruptibility-state into the guest-state area in the
3486 * VMCS.
3487 *
3488 * @returns VBox status code.
3489 * @param pVCpu Pointer to the VMCPU.
3490 * @param uIntrState The interruptibility-state to set.
3491 */
3492static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3493{
3494 NOREF(pVCpu);
3495 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3496 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3497 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3498 AssertRCReturn(rc, rc);
3499 return rc;
3500}
3501
3502
3503/**
3504 * Loads the guest's RIP into the guest-state area in the VMCS.
3505 *
3506 * @returns VBox status code.
3507 * @param pVCpu Pointer to the VMCPU.
3508 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3509 * out-of-sync. Make sure to update the required fields
3510 * before using them.
3511 *
3512 * @remarks No-long-jump zone!!!
3513 */
3514static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3515{
3516 int rc = VINF_SUCCESS;
3517 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3518 {
3519 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3520 AssertRCReturn(rc, rc);
3521
3522 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3523 Log4(("Load: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pMixedCtx->rip, HMCPU_CF_VALUE(pVCpu)));
3524 }
3525 return rc;
3526}
3527
3528
3529/**
3530 * Loads the guest's RSP into the guest-state area in the VMCS.
3531 *
3532 * @returns VBox status code.
3533 * @param pVCpu Pointer to the VMCPU.
3534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3535 * out-of-sync. Make sure to update the required fields
3536 * before using them.
3537 *
3538 * @remarks No-long-jump zone!!!
3539 */
3540static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3541{
3542 int rc = VINF_SUCCESS;
3543 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3544 {
3545 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3546 AssertRCReturn(rc, rc);
3547
3548 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3549 Log4(("Load: VMX_VMCS_GUEST_RSP=%#RX64\n", pMixedCtx->rsp));
3550 }
3551 return rc;
3552}
3553
3554
3555/**
3556 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3557 *
3558 * @returns VBox status code.
3559 * @param pVCpu Pointer to the VMCPU.
3560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3561 * out-of-sync. Make sure to update the required fields
3562 * before using them.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3570 {
3571 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3572 Let us assert it as such and use 32-bit VMWRITE. */
3573 Assert(!(pMixedCtx->rflags.u64 >> 32));
3574 X86EFLAGS Eflags = pMixedCtx->eflags;
3575 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3576 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3577
3578 /*
3579 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3580 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3581 */
3582 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3583 {
3584 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3585 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3586 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3587 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3588 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3589 }
3590
3591 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3592 AssertRCReturn(rc, rc);
3593
3594 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3595 Log4(("Load: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", Eflags.u32));
3596 }
3597 return rc;
3598}
3599
3600
3601/**
3602 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3603 *
3604 * @returns VBox status code.
3605 * @param pVCpu Pointer to the VMCPU.
3606 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3607 * out-of-sync. Make sure to update the required fields
3608 * before using them.
3609 *
3610 * @remarks No-long-jump zone!!!
3611 */
3612DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3613{
3614 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3615 AssertRCReturn(rc, rc);
3616 rc = hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3617 AssertRCReturn(rc, rc);
3618 rc = hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3619 AssertRCReturn(rc, rc);
3620 return rc;
3621}
3622
3623
3624/**
3625 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3626 * CR0 is partially shared with the host and we have to consider the FPU bits.
3627 *
3628 * @returns VBox status code.
3629 * @param pVM Pointer to the VM.
3630 * @param pVCpu Pointer to the VMCPU.
3631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3632 * out-of-sync. Make sure to update the required fields
3633 * before using them.
3634 *
3635 * @remarks No-long-jump zone!!!
3636 */
3637static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3638{
3639 /*
3640 * Guest CR0.
3641 * Guest FPU.
3642 */
3643 int rc = VINF_SUCCESS;
3644 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3645 {
3646 Assert(!(pMixedCtx->cr0 >> 32));
3647 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3648 PVM pVM = pVCpu->CTX_SUFF(pVM);
3649
3650 /* The guest's view (read access) of its CR0 is unblemished. */
3651 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3652 AssertRCReturn(rc, rc);
3653 Log4(("Load: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", u32GuestCR0));
3654
3655 /* Setup VT-x's view of the guest CR0. */
3656 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3657 if (pVM->hm.s.fNestedPaging)
3658 {
3659 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3660 {
3661 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3662 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3663 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3664 }
3665 else
3666 {
3667 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3668 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3669 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3670 }
3671
3672 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3673 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3674 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3675
3676 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3677 AssertRCReturn(rc, rc);
3678 }
3679 else
3680 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3681
3682 /*
3683 * Guest FPU bits.
3684 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3685 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3686 */
3687 u32GuestCR0 |= X86_CR0_NE;
3688 bool fInterceptNM = false;
3689 if (CPUMIsGuestFPUStateActive(pVCpu))
3690 {
3691 fInterceptNM = false; /* Guest FPU active, no need to VM-exit on #NM. */
3692 /* The guest should still get #NM exceptions when it expects it to, so we should not clear TS & MP bits here.
3693 We're only concerned about -us- not intercepting #NMs when the guest-FPU is active. Not the guest itself! */
3694 }
3695 else
3696 {
3697 fInterceptNM = true; /* Guest FPU inactive, VM-exit on #NM for lazy FPU loading. */
3698 u32GuestCR0 |= X86_CR0_TS /* Guest can task switch quickly and do lazy FPU syncing. */
3699 | X86_CR0_MP; /* FWAIT/WAIT should not ignore CR0.TS and should generate #NM. */
3700 }
3701
3702 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3703 bool fInterceptMF = false;
3704 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3705 fInterceptMF = true;
3706
3707 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3708 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3709 {
3710 Assert(PDMVmmDevHeapIsEnabled(pVM));
3711 Assert(pVM->hm.s.vmx.pRealModeTSS);
3712 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3713 fInterceptNM = true;
3714 fInterceptMF = true;
3715 }
3716 else
3717 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3718
3719 if (fInterceptNM)
3720 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_NM);
3721 else
3722 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_NM);
3723
3724 if (fInterceptMF)
3725 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3726 else
3727 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_MF);
3728
3729 /* Additional intercepts for debugging, define these yourself explicitly. */
3730#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3731 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3732 | RT_BIT(X86_XCPT_BP)
3733 | RT_BIT(X86_XCPT_DB)
3734 | RT_BIT(X86_XCPT_DE)
3735 | RT_BIT(X86_XCPT_NM)
3736 | RT_BIT(X86_XCPT_TS)
3737 | RT_BIT(X86_XCPT_UD)
3738 | RT_BIT(X86_XCPT_NP)
3739 | RT_BIT(X86_XCPT_SS)
3740 | RT_BIT(X86_XCPT_GP)
3741 | RT_BIT(X86_XCPT_PF)
3742 | RT_BIT(X86_XCPT_MF)
3743 ;
3744#elif defined(HMVMX_ALWAYS_TRAP_PF)
3745 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3746#endif
3747
3748 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3749
3750 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3751 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3752 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3753 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3754 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3755 else
3756 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3757
3758 u32GuestCR0 |= uSetCR0;
3759 u32GuestCR0 &= uZapCR0;
3760 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3761
3762 /* Write VT-x's view of the guest CR0 into the VMCS and update the exception bitmap. */
3763 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3764 AssertRCReturn(rc, rc);
3765 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3766 AssertRCReturn(rc, rc);
3767 Log4(("Load: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", u32GuestCR0, uSetCR0, uZapCR0));
3768
3769 /*
3770 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3771 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3772 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3773 */
3774 uint32_t u32CR0Mask = 0;
3775 u32CR0Mask = X86_CR0_PE
3776 | X86_CR0_NE
3777 | X86_CR0_WP
3778 | X86_CR0_PG
3779 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3780 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3781 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3782
3783 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3784 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3785 * and @bugref{6944}. */
3786#if 0
3787 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3788 u32CR0Mask &= ~X86_CR0_PE;
3789#endif
3790 if (pVM->hm.s.fNestedPaging)
3791 u32CR0Mask &= ~X86_CR0_WP;
3792
3793 /* If the guest FPU state is active, don't need to VM-exit on writes to FPU related bits in CR0. */
3794 if (fInterceptNM)
3795 {
3796 u32CR0Mask |= X86_CR0_TS
3797 | X86_CR0_MP;
3798 }
3799
3800 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3801 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3802 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3803 AssertRCReturn(rc, rc);
3804 Log4(("Load: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", u32CR0Mask));
3805
3806 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3807 }
3808 return rc;
3809}
3810
3811
3812/**
3813 * Loads the guest control registers (CR3, CR4) into the guest-state area
3814 * in the VMCS.
3815 *
3816 * @returns VBox status code.
3817 * @param pVM Pointer to the VM.
3818 * @param pVCpu Pointer to the VMCPU.
3819 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3820 * out-of-sync. Make sure to update the required fields
3821 * before using them.
3822 *
3823 * @remarks No-long-jump zone!!!
3824 */
3825static int hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3826{
3827 int rc = VINF_SUCCESS;
3828 PVM pVM = pVCpu->CTX_SUFF(pVM);
3829
3830 /*
3831 * Guest CR2.
3832 * It's always loaded in the assembler code. Nothing to do here.
3833 */
3834
3835 /*
3836 * Guest CR3.
3837 */
3838 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3839 {
3840 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3841 if (pVM->hm.s.fNestedPaging)
3842 {
3843 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3844
3845 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3846 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3847 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3848 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3849
3850 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3851 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3852 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3853
3854 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3855 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3856 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x3f) == 0, /* Bits 6:11 MBZ. */
3857 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3858
3859 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3860 AssertRCReturn(rc, rc);
3861 Log4(("Load: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862
3863 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3864 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3865 {
3866 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3867 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3868 {
3869 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]); AssertRCReturn(rc, rc);
3870 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
3871 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
3872 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
3873 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
3874 }
3875
3876 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3877 have Unrestricted Execution to handle the guest when it's not using paging. */
3878 GCPhysGuestCR3 = pMixedCtx->cr3;
3879 }
3880 else
3881 {
3882 /*
3883 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
3884 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
3885 * EPT takes care of translating it to host-physical addresses.
3886 */
3887 RTGCPHYS GCPhys;
3888 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3889 Assert(PDMVmmDevHeapIsEnabled(pVM));
3890
3891 /* We obtain it here every time as the guest could have relocated this PCI region. */
3892 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3893 AssertRCReturn(rc, rc);
3894
3895 GCPhysGuestCR3 = GCPhys;
3896 }
3897
3898 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RGv (GstN)\n", GCPhysGuestCR3));
3899 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3900 }
3901 else
3902 {
3903 /* Non-nested paging case, just use the hypervisor's CR3. */
3904 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3905
3906 Log4(("Load: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3907 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3908 }
3909 AssertRCReturn(rc, rc);
3910
3911 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
3912 }
3913
3914 /*
3915 * Guest CR4.
3916 */
3917 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
3918 {
3919 Assert(!(pMixedCtx->cr4 >> 32));
3920 uint32_t u32GuestCR4 = pMixedCtx->cr4;
3921
3922 /* The guest's view of its CR4 is unblemished. */
3923 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
3924 AssertRCReturn(rc, rc);
3925 Log4(("Load: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", u32GuestCR4));
3926
3927 /* Setup VT-x's view of the guest CR4. */
3928 /*
3929 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
3930 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3931 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3932 */
3933 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3934 {
3935 Assert(pVM->hm.s.vmx.pRealModeTSS);
3936 Assert(PDMVmmDevHeapIsEnabled(pVM));
3937 u32GuestCR4 &= ~X86_CR4_VME;
3938 }
3939
3940 if (pVM->hm.s.fNestedPaging)
3941 {
3942 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3943 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3944 {
3945 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3946 u32GuestCR4 |= X86_CR4_PSE;
3947 /* Our identity mapping is a 32-bit page directory. */
3948 u32GuestCR4 &= ~X86_CR4_PAE;
3949 }
3950 /* else use guest CR4.*/
3951 }
3952 else
3953 {
3954 /*
3955 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3956 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3957 */
3958 switch (pVCpu->hm.s.enmShadowMode)
3959 {
3960 case PGMMODE_REAL: /* Real-mode. */
3961 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3962 case PGMMODE_32_BIT: /* 32-bit paging. */
3963 {
3964 u32GuestCR4 &= ~X86_CR4_PAE;
3965 break;
3966 }
3967
3968 case PGMMODE_PAE: /* PAE paging. */
3969 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3970 {
3971 u32GuestCR4 |= X86_CR4_PAE;
3972 break;
3973 }
3974
3975 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3976 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3977#ifdef VBOX_ENABLE_64_BITS_GUESTS
3978 break;
3979#endif
3980 default:
3981 AssertFailed();
3982 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
3983 }
3984 }
3985
3986 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
3987 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3988 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
3989 u32GuestCR4 |= uSetCR4;
3990 u32GuestCR4 &= uZapCR4;
3991
3992 /* Write VT-x's view of the guest CR4 into the VMCS. */
3993 Log4(("Load: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", u32GuestCR4, uSetCR4, uZapCR4));
3994 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
3995 AssertRCReturn(rc, rc);
3996
3997 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
3998 uint32_t u32CR4Mask = 0;
3999 u32CR4Mask = X86_CR4_VME
4000 | X86_CR4_PAE
4001 | X86_CR4_PGE
4002 | X86_CR4_PSE
4003 | X86_CR4_VMXE;
4004 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4005 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4006 AssertRCReturn(rc, rc);
4007
4008 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4009 }
4010 return rc;
4011}
4012
4013
4014/**
4015 * Loads the guest debug registers into the guest-state area in the VMCS.
4016 * This also sets up whether #DB and MOV DRx accesses cause VM-exits.
4017 *
4018 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4019 *
4020 * @returns VBox status code.
4021 * @param pVCpu Pointer to the VMCPU.
4022 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4023 * out-of-sync. Make sure to update the required fields
4024 * before using them.
4025 *
4026 * @remarks No-long-jump zone!!!
4027 */
4028static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4029{
4030 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4031 return VINF_SUCCESS;
4032
4033#ifdef VBOX_STRICT
4034 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4035 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4036 {
4037 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4038 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4039 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4040 }
4041#endif
4042
4043 int rc;
4044 PVM pVM = pVCpu->CTX_SUFF(pVM);
4045 bool fInterceptDB = false;
4046 bool fInterceptMovDRx = false;
4047 if ( pVCpu->hm.s.fSingleInstruction
4048 || DBGFIsStepping(pVCpu))
4049 {
4050 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4051 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4052 {
4053 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4054 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4055 AssertRCReturn(rc, rc);
4056 Assert(fInterceptDB == false);
4057 }
4058 else
4059 {
4060 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4061 pVCpu->hm.s.fClearTrapFlag = true;
4062 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4063 fInterceptDB = true;
4064 }
4065 }
4066
4067 if ( fInterceptDB
4068 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4069 {
4070 /*
4071 * Use the combined guest and host DRx values found in the hypervisor
4072 * register set because the debugger has breakpoints active or someone
4073 * is single stepping on the host side without a monitor trap flag.
4074 *
4075 * Note! DBGF expects a clean DR6 state before executing guest code.
4076 */
4077#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4078 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4079 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4080 {
4081 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4082 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4083 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4084 }
4085 else
4086#endif
4087 if (!CPUMIsHyperDebugStateActive(pVCpu))
4088 {
4089 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4090 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4091 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4092 }
4093
4094 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4095 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4096 AssertRCReturn(rc, rc);
4097
4098 pVCpu->hm.s.fUsingHyperDR7 = true;
4099 fInterceptDB = true;
4100 fInterceptMovDRx = true;
4101 }
4102 else
4103 {
4104 /*
4105 * If the guest has enabled debug registers, we need to load them prior to
4106 * executing guest code so they'll trigger at the right time.
4107 */
4108 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4109 {
4110#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4111 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4112 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4113 {
4114 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4115 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4116 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4117 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4118 }
4119 else
4120#endif
4121 if (!CPUMIsGuestDebugStateActive(pVCpu))
4122 {
4123 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4124 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4125 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4126 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4127 }
4128 Assert(!fInterceptDB);
4129 Assert(!fInterceptMovDRx);
4130 }
4131 /*
4132 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4133 * must intercept #DB in order to maintain a correct DR6 guest value.
4134 */
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4136 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4137 && !CPUMIsGuestDebugStateActive(pVCpu))
4138#else
4139 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4140#endif
4141 {
4142 fInterceptMovDRx = true;
4143 fInterceptDB = true;
4144 }
4145
4146 /* Update guest DR7. */
4147 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4148 AssertRCReturn(rc, rc);
4149
4150 pVCpu->hm.s.fUsingHyperDR7 = false;
4151 }
4152
4153 /*
4154 * Update the exception bitmap regarding intercepting #DB generated by the guest.
4155 */
4156 if (fInterceptDB)
4157 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_DB);
4158 else if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4159 {
4160#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
4161 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
4162#endif
4163 }
4164 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
4165 AssertRCReturn(rc, rc);
4166
4167 /*
4168 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4169 */
4170 if (fInterceptMovDRx)
4171 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4172 else
4173 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4174 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4175 AssertRCReturn(rc, rc);
4176
4177 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4178 return VINF_SUCCESS;
4179}
4180
4181
4182#ifdef VBOX_STRICT
4183/**
4184 * Strict function to validate segment registers.
4185 *
4186 * @remarks ASSUMES CR0 is up to date.
4187 */
4188static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4189{
4190 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4191 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4192 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4193 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4194 && ( !CPUMIsGuestInRealModeEx(pCtx)
4195 && !CPUMIsGuestInV86ModeEx(pCtx)))
4196 {
4197 /* Protected mode checks */
4198 /* CS */
4199 Assert(pCtx->cs.Attr.n.u1Present);
4200 Assert(!(pCtx->cs.Attr.u & 0xf00));
4201 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4202 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4203 || !(pCtx->cs.Attr.n.u1Granularity));
4204 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4205 || (pCtx->cs.Attr.n.u1Granularity));
4206 /* CS cannot be loaded with NULL in protected mode. */
4207 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS?!? */
4208 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4209 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4210 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4211 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4212 else
4213 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4214 /* SS */
4215 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4216 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4217 if ( !(pCtx->cr0 & X86_CR0_PE)
4218 || pCtx->cs.Attr.n.u4Type == 3)
4219 {
4220 Assert(!pCtx->ss.Attr.n.u2Dpl);
4221 }
4222 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4223 {
4224 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4225 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4226 Assert(pCtx->ss.Attr.n.u1Present);
4227 Assert(!(pCtx->ss.Attr.u & 0xf00));
4228 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4229 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4230 || !(pCtx->ss.Attr.n.u1Granularity));
4231 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4232 || (pCtx->ss.Attr.n.u1Granularity));
4233 }
4234 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4235 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4236 {
4237 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4238 Assert(pCtx->ds.Attr.n.u1Present);
4239 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4240 Assert(!(pCtx->ds.Attr.u & 0xf00));
4241 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4242 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4243 || !(pCtx->ds.Attr.n.u1Granularity));
4244 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4245 || (pCtx->ds.Attr.n.u1Granularity));
4246 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4247 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4248 }
4249 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4250 {
4251 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4252 Assert(pCtx->es.Attr.n.u1Present);
4253 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4254 Assert(!(pCtx->es.Attr.u & 0xf00));
4255 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4256 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4257 || !(pCtx->es.Attr.n.u1Granularity));
4258 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4259 || (pCtx->es.Attr.n.u1Granularity));
4260 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4261 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4262 }
4263 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4264 {
4265 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4266 Assert(pCtx->fs.Attr.n.u1Present);
4267 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4268 Assert(!(pCtx->fs.Attr.u & 0xf00));
4269 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4270 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4271 || !(pCtx->fs.Attr.n.u1Granularity));
4272 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4273 || (pCtx->fs.Attr.n.u1Granularity));
4274 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4275 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4276 }
4277 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4278 {
4279 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4280 Assert(pCtx->gs.Attr.n.u1Present);
4281 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4282 Assert(!(pCtx->gs.Attr.u & 0xf00));
4283 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4284 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4285 || !(pCtx->gs.Attr.n.u1Granularity));
4286 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4287 || (pCtx->gs.Attr.n.u1Granularity));
4288 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4289 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4290 }
4291 /* 64-bit capable CPUs. */
4292# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4293 if (HMVMX_IS_64BIT_HOST_MODE())
4294 {
4295 Assert(!(pCtx->cs.u64Base >> 32));
4296 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4297 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4298 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4299 }
4300# endif
4301 }
4302 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4303 || ( CPUMIsGuestInRealModeEx(pCtx)
4304 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4305 {
4306 /* Real and v86 mode checks. */
4307 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4308 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4309 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4310 {
4311 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4312 }
4313 else
4314 {
4315 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4316 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4317 }
4318
4319 /* CS */
4320 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4321 Assert(pCtx->cs.u32Limit == 0xffff);
4322 Assert(u32CSAttr == 0xf3);
4323 /* SS */
4324 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4325 Assert(pCtx->ss.u32Limit == 0xffff);
4326 Assert(u32SSAttr == 0xf3);
4327 /* DS */
4328 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4329 Assert(pCtx->ds.u32Limit == 0xffff);
4330 Assert(u32DSAttr == 0xf3);
4331 /* ES */
4332 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4333 Assert(pCtx->es.u32Limit == 0xffff);
4334 Assert(u32ESAttr == 0xf3);
4335 /* FS */
4336 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4337 Assert(pCtx->fs.u32Limit == 0xffff);
4338 Assert(u32FSAttr == 0xf3);
4339 /* GS */
4340 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4341 Assert(pCtx->gs.u32Limit == 0xffff);
4342 Assert(u32GSAttr == 0xf3);
4343 /* 64-bit capable CPUs. */
4344# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4345 if (HMVMX_IS_64BIT_HOST_MODE())
4346 {
4347 Assert(!(pCtx->cs.u64Base >> 32));
4348 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4349 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4350 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4351 }
4352# endif
4353 }
4354}
4355#endif /* VBOX_STRICT */
4356
4357
4358/**
4359 * Writes a guest segment register into the guest-state area in the VMCS.
4360 *
4361 * @returns VBox status code.
4362 * @param pVCpu Pointer to the VMCPU.
4363 * @param idxSel Index of the selector in the VMCS.
4364 * @param idxLimit Index of the segment limit in the VMCS.
4365 * @param idxBase Index of the segment base in the VMCS.
4366 * @param idxAccess Index of the access rights of the segment in the VMCS.
4367 * @param pSelReg Pointer to the segment selector.
4368 *
4369 * @remarks No-long-jump zone!!!
4370 */
4371static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4372 uint32_t idxAccess, PCPUMSELREG pSelReg)
4373{
4374 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4375 AssertRCReturn(rc, rc);
4376 rc = VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4377 AssertRCReturn(rc, rc);
4378 rc = VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4379 AssertRCReturn(rc, rc);
4380
4381 uint32_t u32Access = pSelReg->Attr.u;
4382 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4383 {
4384 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4385 u32Access = 0xf3;
4386 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4387 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4388 }
4389 else
4390 {
4391 /*
4392 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4393 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4394 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4395 * loaded in protected-mode have their attribute as 0.
4396 */
4397 if (!u32Access)
4398 u32Access = X86DESCATTR_UNUSABLE;
4399 }
4400
4401 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4402 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4403 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4404
4405 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4406 AssertRCReturn(rc, rc);
4407 return rc;
4408}
4409
4410
4411/**
4412 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4413 * into the guest-state area in the VMCS.
4414 *
4415 * @returns VBox status code.
4416 * @param pVM Pointer to the VM.
4417 * @param pVCPU Pointer to the VMCPU.
4418 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4419 * out-of-sync. Make sure to update the required fields
4420 * before using them.
4421 *
4422 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4423 * @remarks No-long-jump zone!!!
4424 */
4425static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4426{
4427 int rc = VERR_INTERNAL_ERROR_5;
4428 PVM pVM = pVCpu->CTX_SUFF(pVM);
4429
4430 /*
4431 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4432 */
4433 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4434 {
4435 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4436 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4437 {
4438 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4439 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4440 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4441 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4442 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4443 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4444 }
4445
4446#ifdef VBOX_WITH_REM
4447 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4448 {
4449 Assert(pVM->hm.s.vmx.pRealModeTSS);
4450 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4451 if ( pVCpu->hm.s.vmx.fWasInRealMode
4452 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4453 {
4454 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4455 in real-mode (e.g. OpenBSD 4.0) */
4456 REMFlushTBs(pVM);
4457 Log4(("Load: Switch to protected mode detected!\n"));
4458 pVCpu->hm.s.vmx.fWasInRealMode = false;
4459 }
4460 }
4461#endif
4462 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_CS, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4463 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4464 AssertRCReturn(rc, rc);
4465 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_SS, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4466 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4467 AssertRCReturn(rc, rc);
4468 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_DS, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4469 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4470 AssertRCReturn(rc, rc);
4471 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_ES, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4472 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4473 AssertRCReturn(rc, rc);
4474 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_FS, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4475 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4476 AssertRCReturn(rc, rc);
4477 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_GS, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4478 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4479 AssertRCReturn(rc, rc);
4480
4481#ifdef VBOX_STRICT
4482 /* Validate. */
4483 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4484#endif
4485
4486 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4487 Log4(("Load: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4488 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4489 }
4490
4491 /*
4492 * Guest TR.
4493 */
4494 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4495 {
4496 /*
4497 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4498 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4499 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4500 */
4501 uint16_t u16Sel = 0;
4502 uint32_t u32Limit = 0;
4503 uint64_t u64Base = 0;
4504 uint32_t u32AccessRights = 0;
4505
4506 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4507 {
4508 u16Sel = pMixedCtx->tr.Sel;
4509 u32Limit = pMixedCtx->tr.u32Limit;
4510 u64Base = pMixedCtx->tr.u64Base;
4511 u32AccessRights = pMixedCtx->tr.Attr.u;
4512 }
4513 else
4514 {
4515 Assert(pVM->hm.s.vmx.pRealModeTSS);
4516 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4517
4518 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4519 RTGCPHYS GCPhys;
4520 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4521 AssertRCReturn(rc, rc);
4522
4523 X86DESCATTR DescAttr;
4524 DescAttr.u = 0;
4525 DescAttr.n.u1Present = 1;
4526 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4527
4528 u16Sel = 0;
4529 u32Limit = HM_VTX_TSS_SIZE;
4530 u64Base = GCPhys; /* in real-mode phys = virt. */
4531 u32AccessRights = DescAttr.u;
4532 }
4533
4534 /* Validate. */
4535 Assert(!(u16Sel & RT_BIT(2)));
4536 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4537 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4538 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4539 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4540 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4541 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4542 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4543 Assert( (u32Limit & 0xfff) == 0xfff
4544 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4545 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4546 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4547
4548 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_TR, u16Sel); AssertRCReturn(rc, rc);
4549 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit); AssertRCReturn(rc, rc);
4550 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base); AssertRCReturn(rc, rc);
4551 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights); AssertRCReturn(rc, rc);
4552
4553 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4554 Log4(("Load: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", u64Base));
4555 }
4556
4557 /*
4558 * Guest GDTR.
4559 */
4560 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4561 {
4562 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt); AssertRCReturn(rc, rc);
4563 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt); AssertRCReturn(rc, rc);
4564
4565 /* Validate. */
4566 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4567
4568 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4569 Log4(("Load: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pMixedCtx->gdtr.pGdt));
4570 }
4571
4572 /*
4573 * Guest LDTR.
4574 */
4575 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4576 {
4577 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4578 uint32_t u32Access = 0;
4579 if (!pMixedCtx->ldtr.Attr.u)
4580 u32Access = X86DESCATTR_UNUSABLE;
4581 else
4582 u32Access = pMixedCtx->ldtr.Attr.u;
4583
4584 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_FIELD_LDTR, pMixedCtx->ldtr.Sel); AssertRCReturn(rc, rc);
4585 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit); AssertRCReturn(rc, rc);
4586 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base); AssertRCReturn(rc, rc);
4587 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access); AssertRCReturn(rc, rc);
4588
4589 /* Validate. */
4590 if (!(u32Access & X86DESCATTR_UNUSABLE))
4591 {
4592 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4593 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4594 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4595 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4596 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4597 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4598 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4599 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4600 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4601 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4602 }
4603
4604 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4605 Log4(("Load: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pMixedCtx->ldtr.u64Base));
4606 }
4607
4608 /*
4609 * Guest IDTR.
4610 */
4611 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4612 {
4613 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt); AssertRCReturn(rc, rc);
4614 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt); AssertRCReturn(rc, rc);
4615
4616 /* Validate. */
4617 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4618
4619 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4620 Log4(("Load: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pMixedCtx->idtr.pIdt));
4621 }
4622
4623 return VINF_SUCCESS;
4624}
4625
4626
4627/**
4628 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4629 * areas. These MSRs will automatically be loaded to the host CPU on every
4630 * successful VM entry and stored from the host CPU on every successful VM-exit.
4631 *
4632 * This also creates/updates MSR slots for the host MSRs. The actual host
4633 * MSR values are -not- updated here for performance reasons. See
4634 * hmR0VmxSaveHostMsrs().
4635 *
4636 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4637 *
4638 * @returns VBox status code.
4639 * @param pVCpu Pointer to the VMCPU.
4640 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4641 * out-of-sync. Make sure to update the required fields
4642 * before using them.
4643 *
4644 * @remarks No-long-jump zone!!!
4645 */
4646static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4647{
4648 AssertPtr(pVCpu);
4649 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4650
4651 /*
4652 * MSRs that we use the auto-load/store MSR area in the VMCS.
4653 */
4654 PVM pVM = pVCpu->CTX_SUFF(pVM);
4655 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4656 {
4657#if HC_ARCH_BITS == 32 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4658 if (pVM->hm.s.fAllow64BitGuests)
4659 {
4660 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false /* fUpdateHostMsr */);
4661 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false /* fUpdateHostMsr */);
4662 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false /* fUpdateHostMsr */);
4663 hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false /* fUpdateHostMsr */);
4664# ifdef DEBUG
4665 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4666 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4667 Log4(("Load: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4668# endif
4669 }
4670#endif
4671 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4672 }
4673
4674 /*
4675 * Guest Sysenter MSRs.
4676 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4677 * VM-exits on WRMSRs for these MSRs.
4678 */
4679 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4680 {
4681 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4682 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4683 }
4684
4685 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4686 {
4687 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4688 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4689 }
4690
4691 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4692 {
4693 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4694 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4695 }
4696
4697 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4698 {
4699#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4700 if ( HMVMX_IS_64BIT_HOST_MODE()
4701 && pVM->hm.s.vmx.fSupportsVmcsEfer
4702 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx)) /* Not really needed here, but avoids a VM-write as a nested guest. */
4703 {
4704 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4705 AssertRCReturn(rc,rc);
4706 Log4(("Load: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pMixedCtx->msrEFER));
4707 }
4708#endif
4709 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4710 }
4711
4712 return VINF_SUCCESS;
4713}
4714
4715
4716/**
4717 * Loads the guest activity state into the guest-state area in the VMCS.
4718 *
4719 * @returns VBox status code.
4720 * @param pVCpu Pointer to the VMCPU.
4721 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4722 * out-of-sync. Make sure to update the required fields
4723 * before using them.
4724 *
4725 * @remarks No-long-jump zone!!!
4726 */
4727static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pCtx)
4728{
4729 NOREF(pCtx);
4730 /** @todo See if we can make use of other states, e.g.
4731 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4732 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4733 {
4734 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4735 AssertRCReturn(rc, rc);
4736
4737 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4738 }
4739 return VINF_SUCCESS;
4740}
4741
4742
4743/**
4744 * Sets up the appropriate function to run guest code.
4745 *
4746 * @returns VBox status code.
4747 * @param pVCpu Pointer to the VMCPU.
4748 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4749 * out-of-sync. Make sure to update the required fields
4750 * before using them.
4751 *
4752 * @remarks No-long-jump zone!!!
4753 */
4754static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4755{
4756 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4757 {
4758#ifndef VBOX_ENABLE_64_BITS_GUESTS
4759 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4760#endif
4761 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4762#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4763 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4764 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4765 {
4766 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4767 {
4768 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4769 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4770 | HM_CHANGED_VMX_EXIT_CTLS
4771 | HM_CHANGED_VMX_ENTRY_CTLS
4772 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4773 }
4774 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4775 }
4776#else
4777 /* 64-bit host or hybrid host. */
4778 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4779#endif
4780 }
4781 else
4782 {
4783 /* Guest is not in long mode, use the 32-bit handler. */
4784#if HC_ARCH_BITS == 32 && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
4785 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4786 {
4787 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4788 {
4789 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4790 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT
4791 | HM_CHANGED_VMX_EXIT_CTLS
4792 | HM_CHANGED_VMX_ENTRY_CTLS
4793 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4794 }
4795 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4796 }
4797#else
4798 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4799#endif
4800 }
4801 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4802 return VINF_SUCCESS;
4803}
4804
4805
4806/**
4807 * Wrapper for running the guest code in VT-x.
4808 *
4809 * @returns VBox strict status code.
4810 * @param pVM Pointer to the VM.
4811 * @param pVCpu Pointer to the VMCPU.
4812 * @param pCtx Pointer to the guest-CPU context.
4813 *
4814 * @remarks No-long-jump zone!!!
4815 */
4816DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4817{
4818 /*
4819 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
4820 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
4821 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
4822 */
4823 const bool fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4824 /** @todo Add stats for resume vs launch. */
4825#ifdef VBOX_WITH_KERNEL_USING_XMM
4826 return HMR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4827#else
4828 return pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4829#endif
4830}
4831
4832
4833/**
4834 * Reports world-switch error and dumps some useful debug info.
4835 *
4836 * @param pVM Pointer to the VM.
4837 * @param pVCpu Pointer to the VMCPU.
4838 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4839 * @param pCtx Pointer to the guest-CPU context.
4840 * @param pVmxTransient Pointer to the VMX transient structure (only
4841 * exitReason updated).
4842 */
4843static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4844{
4845 Assert(pVM);
4846 Assert(pVCpu);
4847 Assert(pCtx);
4848 Assert(pVmxTransient);
4849 HMVMX_ASSERT_PREEMPT_SAFE();
4850
4851 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
4852 switch (rcVMRun)
4853 {
4854 case VERR_VMX_INVALID_VMXON_PTR:
4855 AssertFailed();
4856 break;
4857 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4858 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4859 {
4860 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4861 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4862 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4863 AssertRC(rc);
4864
4865 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4866 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4867 Cannot do it here as we may have been long preempted. */
4868
4869#ifdef VBOX_STRICT
4870 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4871 pVmxTransient->uExitReason));
4872 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4873 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4874 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4875 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4876 else
4877 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4878 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4879 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4880
4881 /* VMX control bits. */
4882 uint32_t u32Val;
4883 uint64_t u64Val;
4884 HMVMXHCUINTREG uHCReg;
4885 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4886 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4887 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4888 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4889 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4890 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4891 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4892 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4893 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4894 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4895 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
4896 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
4897 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
4898 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
4899 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
4900 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
4901 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
4902 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
4903 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
4904 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
4905 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
4906 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
4907 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4908 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
4909 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
4910 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
4911 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
4912 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
4913 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
4914 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
4915 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
4916 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
4917 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
4918 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
4919 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
4920 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4921 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
4922 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
4923 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
4924 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
4925 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
4926 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
4927
4928 /* Guest bits. */
4929 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
4930 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
4931 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
4932 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
4933 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
4934 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
4935 rc = VMXReadVmcs32(VMX_VMCS16_GUEST_FIELD_VPID, &u32Val); AssertRC(rc);
4936 Log4(("VMX_VMCS16_GUEST_FIELD_VPID %u\n", u32Val));
4937
4938 /* Host bits. */
4939 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
4940 Log4(("Host CR0 %#RHr\n", uHCReg));
4941 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
4942 Log4(("Host CR3 %#RHr\n", uHCReg));
4943 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
4944 Log4(("Host CR4 %#RHr\n", uHCReg));
4945
4946 RTGDTR HostGdtr;
4947 PCX86DESCHC pDesc;
4948 ASMGetGDTR(&HostGdtr);
4949 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_CS, &u32Val); AssertRC(rc);
4950 Log4(("Host CS %#08x\n", u32Val));
4951 if (u32Val < HostGdtr.cbGdt)
4952 {
4953 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4954 HMR0DumpDescriptor(pDesc, u32Val, "CS: ");
4955 }
4956
4957 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_DS, &u32Val); AssertRC(rc);
4958 Log4(("Host DS %#08x\n", u32Val));
4959 if (u32Val < HostGdtr.cbGdt)
4960 {
4961 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4962 HMR0DumpDescriptor(pDesc, u32Val, "DS: ");
4963 }
4964
4965 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_ES, &u32Val); AssertRC(rc);
4966 Log4(("Host ES %#08x\n", u32Val));
4967 if (u32Val < HostGdtr.cbGdt)
4968 {
4969 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4970 HMR0DumpDescriptor(pDesc, u32Val, "ES: ");
4971 }
4972
4973 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_FS, &u32Val); AssertRC(rc);
4974 Log4(("Host FS %#08x\n", u32Val));
4975 if (u32Val < HostGdtr.cbGdt)
4976 {
4977 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4978 HMR0DumpDescriptor(pDesc, u32Val, "FS: ");
4979 }
4980
4981 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_GS, &u32Val); AssertRC(rc);
4982 Log4(("Host GS %#08x\n", u32Val));
4983 if (u32Val < HostGdtr.cbGdt)
4984 {
4985 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4986 HMR0DumpDescriptor(pDesc, u32Val, "GS: ");
4987 }
4988
4989 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_SS, &u32Val); AssertRC(rc);
4990 Log4(("Host SS %#08x\n", u32Val));
4991 if (u32Val < HostGdtr.cbGdt)
4992 {
4993 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
4994 HMR0DumpDescriptor(pDesc, u32Val, "SS: ");
4995 }
4996
4997 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FIELD_TR, &u32Val); AssertRC(rc);
4998 Log4(("Host TR %#08x\n", u32Val));
4999 if (u32Val < HostGdtr.cbGdt)
5000 {
5001 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5002 HMR0DumpDescriptor(pDesc, u32Val, "TR: ");
5003 }
5004
5005 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5006 Log4(("Host TR Base %#RHv\n", uHCReg));
5007 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5008 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5009 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5010 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5011 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5012 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5013 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5014 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5015 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5016 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5017 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5018 Log4(("Host RSP %#RHv\n", uHCReg));
5019 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5020 Log4(("Host RIP %#RHv\n", uHCReg));
5021# if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5022 if (HMVMX_IS_64BIT_HOST_MODE())
5023 {
5024 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5025 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5026 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5027 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5028 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5029 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5030 }
5031# endif
5032#endif /* VBOX_STRICT */
5033 break;
5034 }
5035
5036 default:
5037 /* Impossible */
5038 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5039 break;
5040 }
5041 NOREF(pVM); NOREF(pCtx);
5042}
5043
5044
5045#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
5046#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5047# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5048#endif
5049#ifdef VBOX_STRICT
5050static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5051{
5052 switch (idxField)
5053 {
5054 case VMX_VMCS_GUEST_RIP:
5055 case VMX_VMCS_GUEST_RSP:
5056 case VMX_VMCS_GUEST_SYSENTER_EIP:
5057 case VMX_VMCS_GUEST_SYSENTER_ESP:
5058 case VMX_VMCS_GUEST_GDTR_BASE:
5059 case VMX_VMCS_GUEST_IDTR_BASE:
5060 case VMX_VMCS_GUEST_CS_BASE:
5061 case VMX_VMCS_GUEST_DS_BASE:
5062 case VMX_VMCS_GUEST_ES_BASE:
5063 case VMX_VMCS_GUEST_FS_BASE:
5064 case VMX_VMCS_GUEST_GS_BASE:
5065 case VMX_VMCS_GUEST_SS_BASE:
5066 case VMX_VMCS_GUEST_LDTR_BASE:
5067 case VMX_VMCS_GUEST_TR_BASE:
5068 case VMX_VMCS_GUEST_CR3:
5069 return true;
5070 }
5071 return false;
5072}
5073
5074static bool hmR0VmxIsValidReadField(uint32_t idxField)
5075{
5076 switch (idxField)
5077 {
5078 /* Read-only fields. */
5079 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5080 return true;
5081 }
5082 /* Remaining readable fields should also be writable. */
5083 return hmR0VmxIsValidWriteField(idxField);
5084}
5085#endif /* VBOX_STRICT */
5086
5087
5088/**
5089 * Executes the specified handler in 64-bit mode.
5090 *
5091 * @returns VBox status code.
5092 * @param pVM Pointer to the VM.
5093 * @param pVCpu Pointer to the VMCPU.
5094 * @param pCtx Pointer to the guest CPU context.
5095 * @param enmOp The operation to perform.
5096 * @param cbParam Number of parameters.
5097 * @param paParam Array of 32-bit parameters.
5098 */
5099VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp, uint32_t cbParam,
5100 uint32_t *paParam)
5101{
5102 int rc, rc2;
5103 PHMGLOBALCPUINFO pCpu;
5104 RTHCPHYS HCPhysCpuPage;
5105 RTCCUINTREG uOldEflags;
5106
5107 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5108 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5109 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5110 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5111
5112#ifdef VBOX_STRICT
5113 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5114 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5115
5116 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5117 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5118#endif
5119
5120 /* Disable interrupts. */
5121 uOldEflags = ASMIntDisableFlags();
5122
5123#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5124 RTCPUID idHostCpu = RTMpCpuId();
5125 CPUMR0SetLApic(pVCpu, idHostCpu);
5126#endif
5127
5128 pCpu = HMR0GetCurrentCpu();
5129 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5130
5131 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5132 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5133
5134 /* Leave VMX Root Mode. */
5135 VMXDisable();
5136
5137 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5138
5139 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5140 CPUMSetHyperEIP(pVCpu, enmOp);
5141 for (int i = (int)cbParam - 1; i >= 0; i--)
5142 CPUMPushHyper(pVCpu, paParam[i]);
5143
5144 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5145
5146 /* Call the switcher. */
5147 rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5148 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5149
5150 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5151 /* Make sure the VMX instructions don't cause #UD faults. */
5152 ASMSetCR4(ASMGetCR4() | X86_CR4_VMXE);
5153
5154 /* Re-enter VMX Root Mode */
5155 rc2 = VMXEnable(HCPhysCpuPage);
5156 if (RT_FAILURE(rc2))
5157 {
5158 ASMSetCR4(ASMGetCR4() & ~X86_CR4_VMXE);
5159 ASMSetFlags(uOldEflags);
5160 return rc2;
5161 }
5162
5163 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5164 AssertRC(rc2);
5165 Assert(!(ASMGetFlags() & X86_EFL_IF));
5166 ASMSetFlags(uOldEflags);
5167 return rc;
5168}
5169
5170
5171/**
5172 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5173 * supporting 64-bit guests.
5174 *
5175 * @returns VBox status code.
5176 * @param fResume Whether to VMLAUNCH or VMRESUME.
5177 * @param pCtx Pointer to the guest-CPU context.
5178 * @param pCache Pointer to the VMCS cache.
5179 * @param pVM Pointer to the VM.
5180 * @param pVCpu Pointer to the VMCPU.
5181 */
5182DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5183{
5184 uint32_t aParam[6];
5185 PHMGLOBALCPUINFO pCpu = NULL;
5186 RTHCPHYS HCPhysCpuPage = 0;
5187 int rc = VERR_INTERNAL_ERROR_5;
5188
5189 pCpu = HMR0GetCurrentCpu();
5190 HCPhysCpuPage = RTR0MemObjGetPagePhysAddr(pCpu->hMemObj, 0);
5191
5192#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5193 pCache->uPos = 1;
5194 pCache->interPD = PGMGetInterPaeCR3(pVM);
5195 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5196#endif
5197
5198#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5199 pCache->TestIn.HCPhysCpuPage = 0;
5200 pCache->TestIn.HCPhysVmcs = 0;
5201 pCache->TestIn.pCache = 0;
5202 pCache->TestOut.HCPhysVmcs = 0;
5203 pCache->TestOut.pCache = 0;
5204 pCache->TestOut.pCtx = 0;
5205 pCache->TestOut.eflags = 0;
5206#endif
5207
5208 aParam[0] = (uint32_t)(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5209 aParam[1] = (uint32_t)(HCPhysCpuPage >> 32); /* Param 1: VMXON physical address - Hi. */
5210 aParam[2] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5211 aParam[3] = (uint32_t)(pVCpu->hm.s.vmx.HCPhysVmcs >> 32); /* Param 2: VMCS physical address - Hi. */
5212 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5213 aParam[5] = 0;
5214
5215#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5216 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5217 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5218#endif
5219 rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, 6, &aParam[0]);
5220
5221#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5222 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5223 Assert(pCtx->dr[4] == 10);
5224 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5225#endif
5226
5227#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5228 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5229 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5230 pVCpu->hm.s.vmx.HCPhysVmcs));
5231 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5232 pCache->TestOut.HCPhysVmcs));
5233 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5234 pCache->TestOut.pCache));
5235 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5236 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5237 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5238 pCache->TestOut.pCtx));
5239 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5240#endif
5241 return rc;
5242}
5243
5244
5245/**
5246 * Initialize the VMCS-Read cache. The VMCS cache is used for 32-bit hosts
5247 * running 64-bit guests (except 32-bit Darwin which runs with 64-bit paging in
5248 * 32-bit mode) for 64-bit fields that cannot be accessed in 32-bit mode. Some
5249 * 64-bit fields -can- be accessed (those that have a 32-bit FULL & HIGH part).
5250 *
5251 * @returns VBox status code.
5252 * @param pVM Pointer to the VM.
5253 * @param pVCpu Pointer to the VMCPU.
5254 */
5255static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5256{
5257#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5258{ \
5259 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5260 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5261 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5262 ++cReadFields; \
5263}
5264
5265 AssertPtr(pVM);
5266 AssertPtr(pVCpu);
5267 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5268 uint32_t cReadFields = 0;
5269
5270 /*
5271 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5272 * and serve to indicate exceptions to the rules.
5273 */
5274
5275 /* Guest-natural selector base fields. */
5276#if 0
5277 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5278 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5279 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5280#endif
5281 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5282 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5283 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5284 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5285 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5286 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5287 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5288 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5289 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5290 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5291 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5292 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5293#if 0
5294 /* Unused natural width guest-state fields. */
5295 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5296 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5297#endif
5298 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5299 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5300
5301 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5302#if 0
5303 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5304 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5305 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5306 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5307 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5308 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5309 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5310 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5311 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5312#endif
5313
5314 /* Natural width guest-state fields. */
5315 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5316#if 0
5317 /* Currently unused field. */
5318 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5319#endif
5320
5321 if (pVM->hm.s.fNestedPaging)
5322 {
5323 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5324 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5325 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5326 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5327 }
5328 else
5329 {
5330 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5331 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5332 }
5333
5334#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5335 return VINF_SUCCESS;
5336}
5337
5338
5339/**
5340 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5341 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5342 * darwin, running 64-bit guests).
5343 *
5344 * @returns VBox status code.
5345 * @param pVCpu Pointer to the VMCPU.
5346 * @param idxField The VMCS field encoding.
5347 * @param u64Val 16, 32 or 64-bit value.
5348 */
5349VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5350{
5351 int rc;
5352 switch (idxField)
5353 {
5354 /*
5355 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5356 */
5357 /* 64-bit Control fields. */
5358 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5359 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5360 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5361 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5362 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5363 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5364 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5365 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5366 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5367 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5368 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5369 case VMX_VMCS64_CTRL_EPTP_FULL:
5370 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5371 /* 64-bit Guest-state fields. */
5372 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5373 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5374 case VMX_VMCS64_GUEST_PAT_FULL:
5375 case VMX_VMCS64_GUEST_EFER_FULL:
5376 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5377 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5378 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5379 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5380 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5381 /* 64-bit Host-state fields. */
5382 case VMX_VMCS64_HOST_FIELD_PAT_FULL:
5383 case VMX_VMCS64_HOST_FIELD_EFER_FULL:
5384 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5385 {
5386 rc = VMXWriteVmcs32(idxField, u64Val);
5387 rc |= VMXWriteVmcs32(idxField + 1, (uint32_t)(u64Val >> 32));
5388 break;
5389 }
5390
5391 /*
5392 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5393 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5394 */
5395 /* Natural-width Guest-state fields. */
5396 case VMX_VMCS_GUEST_CR3:
5397 case VMX_VMCS_GUEST_ES_BASE:
5398 case VMX_VMCS_GUEST_CS_BASE:
5399 case VMX_VMCS_GUEST_SS_BASE:
5400 case VMX_VMCS_GUEST_DS_BASE:
5401 case VMX_VMCS_GUEST_FS_BASE:
5402 case VMX_VMCS_GUEST_GS_BASE:
5403 case VMX_VMCS_GUEST_LDTR_BASE:
5404 case VMX_VMCS_GUEST_TR_BASE:
5405 case VMX_VMCS_GUEST_GDTR_BASE:
5406 case VMX_VMCS_GUEST_IDTR_BASE:
5407 case VMX_VMCS_GUEST_RSP:
5408 case VMX_VMCS_GUEST_RIP:
5409 case VMX_VMCS_GUEST_SYSENTER_ESP:
5410 case VMX_VMCS_GUEST_SYSENTER_EIP:
5411 {
5412 if (!(u64Val >> 32))
5413 {
5414 /* If this field is 64-bit, VT-x will zero out the top bits. */
5415 rc = VMXWriteVmcs32(idxField, (uint32_t)u64Val);
5416 }
5417 else
5418 {
5419 /* Assert that only the 32->64 switcher case should ever come here. */
5420 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5421 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5422 }
5423 break;
5424 }
5425
5426 default:
5427 {
5428 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5429 rc = VERR_INVALID_PARAMETER;
5430 break;
5431 }
5432 }
5433 AssertRCReturn(rc, rc);
5434 return rc;
5435}
5436
5437
5438/**
5439 * Queue up a VMWRITE by using the VMCS write cache. This is only used on 32-bit
5440 * hosts (except darwin) for 64-bit guests.
5441 *
5442 * @param pVCpu Pointer to the VMCPU.
5443 * @param idxField The VMCS field encoding.
5444 * @param u64Val 16, 32 or 64-bit value.
5445 */
5446VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5447{
5448 AssertPtr(pVCpu);
5449 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5450
5451 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5452 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5453
5454 /* Make sure there are no duplicates. */
5455 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5456 {
5457 if (pCache->Write.aField[i] == idxField)
5458 {
5459 pCache->Write.aFieldVal[i] = u64Val;
5460 return VINF_SUCCESS;
5461 }
5462 }
5463
5464 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5465 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5466 pCache->Write.cValidEntries++;
5467 return VINF_SUCCESS;
5468}
5469
5470/* Enable later when the assembly code uses these as callbacks. */
5471#if 0
5472/*
5473 * Loads the VMCS write-cache into the CPU (by executing VMWRITEs).
5474 *
5475 * @param pVCpu Pointer to the VMCPU.
5476 * @param pCache Pointer to the VMCS cache.
5477 *
5478 * @remarks No-long-jump zone!!!
5479 */
5480VMMR0DECL(void) VMXWriteCachedVmcsLoad(PVMCPU pVCpu, PVMCSCACHE pCache)
5481{
5482 AssertPtr(pCache);
5483 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5484 {
5485 int rc = VMXWriteVmcs64(pCache->Write.aField[i], pCache->Write.aFieldVal[i]);
5486 AssertRC(rc);
5487 }
5488 pCache->Write.cValidEntries = 0;
5489}
5490
5491
5492/**
5493 * Stores the VMCS read-cache from the CPU (by executing VMREADs).
5494 *
5495 * @param pVCpu Pointer to the VMCPU.
5496 * @param pCache Pointer to the VMCS cache.
5497 *
5498 * @remarks No-long-jump zone!!!
5499 */
5500VMMR0DECL(void) VMXReadCachedVmcsStore(PVMCPU pVCpu, PVMCSCACHE pCache)
5501{
5502 AssertPtr(pCache);
5503 for (uint32_t i = 0; i < pCache->Read.cValidEntries; i++)
5504 {
5505 int rc = VMXReadVmcs64(pCache->Read.aField[i], &pCache->Read.aFieldVal[i]);
5506 AssertRC(rc);
5507 }
5508}
5509#endif
5510#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL) */
5511
5512
5513/**
5514 * Sets up the usage of TSC-offsetting and updates the VMCS. If offsetting is
5515 * not possible, cause VM-exits on RDTSC(P)s. Also sets up the VMX preemption
5516 * timer.
5517 *
5518 * @returns VBox status code.
5519 * @param pVCpu Pointer to the VMCPU.
5520 *
5521 * @remarks No-long-jump zone!!!
5522 */
5523static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5524{
5525 int rc = VERR_INTERNAL_ERROR_5;
5526 bool fOffsettedTsc = false;
5527 PVM pVM = pVCpu->CTX_SUFF(pVM);
5528 if (pVM->hm.s.vmx.fUsePreemptTimer)
5529 {
5530 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVCpu, &fOffsettedTsc, &pVCpu->hm.s.vmx.u64TSCOffset);
5531
5532 /* Make sure the returned values have sane upper and lower boundaries. */
5533 uint64_t u64CpuHz = SUPGetCpuHzFromGIP(g_pSUPGlobalInfoPage);
5534 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5535 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5536 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5537
5538 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5539 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5540 }
5541 else
5542 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset);
5543
5544 if (fOffsettedTsc)
5545 {
5546 uint64_t u64CurTSC = ASMReadTSC();
5547 if (u64CurTSC + pVCpu->hm.s.vmx.u64TSCOffset >= TMCpuTickGetLastSeen(pVCpu))
5548 {
5549 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5550 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5551
5552 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5553 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5554 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5555 }
5556 else
5557 {
5558 /* VM-exit on RDTSC(P) as we would otherwise pass decreasing TSC values to the guest. */
5559 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5560 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5561 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscInterceptOverFlow);
5562 }
5563 }
5564 else
5565 {
5566 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5567 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5568 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5569 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5570 }
5571}
5572
5573
5574/**
5575 * Determines if an exception is a contributory exception. Contributory
5576 * exceptions are ones which can cause double-faults. Page-fault is
5577 * intentionally not included here as it's a conditional contributory exception.
5578 *
5579 * @returns true if the exception is contributory, false otherwise.
5580 * @param uVector The exception vector.
5581 */
5582DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5583{
5584 switch (uVector)
5585 {
5586 case X86_XCPT_GP:
5587 case X86_XCPT_SS:
5588 case X86_XCPT_NP:
5589 case X86_XCPT_TS:
5590 case X86_XCPT_DE:
5591 return true;
5592 default:
5593 break;
5594 }
5595 return false;
5596}
5597
5598
5599/**
5600 * Sets an event as a pending event to be injected into the guest.
5601 *
5602 * @param pVCpu Pointer to the VMCPU.
5603 * @param u32IntInfo The VM-entry interruption-information field.
5604 * @param cbInstr The VM-entry instruction length in bytes (for software
5605 * interrupts, exceptions and privileged software
5606 * exceptions).
5607 * @param u32ErrCode The VM-entry exception error code.
5608 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5609 * page-fault.
5610 *
5611 * @remarks Statistics counter assumes this is a guest event being injected or
5612 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5613 * always incremented.
5614 */
5615DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5616 RTGCUINTPTR GCPtrFaultAddress)
5617{
5618 Assert(!pVCpu->hm.s.Event.fPending);
5619 pVCpu->hm.s.Event.fPending = true;
5620 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5621 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5622 pVCpu->hm.s.Event.cbInstr = cbInstr;
5623 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5624
5625 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5626}
5627
5628
5629/**
5630 * Sets a double-fault (#DF) exception as pending-for-injection into the VM.
5631 *
5632 * @param pVCpu Pointer to the VMCPU.
5633 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5634 * out-of-sync. Make sure to update the required fields
5635 * before using them.
5636 */
5637DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5638{
5639 NOREF(pMixedCtx);
5640 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5641 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5642 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5643 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5644}
5645
5646
5647/**
5648 * Handle a condition that occurred while delivering an event through the guest
5649 * IDT.
5650 *
5651 * @returns VBox status code (informational error codes included).
5652 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5653 * @retval VINF_HM_DOUBLE_FAULT if a #DF condition was detected and we ought to
5654 * continue execution of the guest which will delivery the #DF.
5655 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5656 *
5657 * @param pVCpu Pointer to the VMCPU.
5658 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5659 * out-of-sync. Make sure to update the required fields
5660 * before using them.
5661 * @param pVmxTransient Pointer to the VMX transient structure.
5662 *
5663 * @remarks No-long-jump zone!!!
5664 */
5665static int hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5666{
5667 int rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5668 AssertRCReturn(rc, rc);
5669 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5670 {
5671 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5672 AssertRCReturn(rc, rc);
5673
5674 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5675 uint32_t uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5676 uint32_t uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5677
5678 typedef enum
5679 {
5680 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
5681 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
5682 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
5683 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
5684 } VMXREFLECTXCPT;
5685
5686 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
5687 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
5688 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5689 {
5690 if (uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
5691 {
5692 enmReflect = VMXREFLECTXCPT_XCPT;
5693#ifdef VBOX_STRICT
5694 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
5695 && uExitVector == X86_XCPT_PF)
5696 {
5697 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5698 }
5699#endif
5700 if ( uExitVector == X86_XCPT_PF
5701 && uIdtVector == X86_XCPT_PF)
5702 {
5703 pVmxTransient->fVectoringPF = true;
5704 Log4(("IDT: vcpu[%RU32] Vectoring #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
5705 }
5706 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
5707 && hmR0VmxIsContributoryXcpt(uExitVector)
5708 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
5709 || uIdtVector == X86_XCPT_PF))
5710 {
5711 enmReflect = VMXREFLECTXCPT_DF;
5712 }
5713 else if (uIdtVector == X86_XCPT_DF)
5714 enmReflect = VMXREFLECTXCPT_TF;
5715 }
5716 else if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5717 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5718 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5719 {
5720 /*
5721 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
5722 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
5723 */
5724 enmReflect = VMXREFLECTXCPT_XCPT;
5725 }
5726 }
5727 else
5728 {
5729 /*
5730 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
5731 * interruption-information will not be valid and we end up here. In such cases, it is sufficient to reflect the
5732 * original exception to the guest after handling the VM-exit.
5733 */
5734 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5735 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
5736 || uIntType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
5737 {
5738 enmReflect = VMXREFLECTXCPT_XCPT;
5739 }
5740 }
5741
5742 switch (enmReflect)
5743 {
5744 case VMXREFLECTXCPT_XCPT:
5745 {
5746 Assert( uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5747 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5748 && uIntType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
5749
5750 uint32_t u32ErrCode = 0;
5751 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5752 {
5753 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5754 AssertRCReturn(rc, rc);
5755 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5756 }
5757
5758 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
5759 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5760 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5761 rc = VINF_SUCCESS;
5762 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
5763 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
5764
5765 break;
5766 }
5767
5768 case VMXREFLECTXCPT_DF:
5769 {
5770 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5771 rc = VINF_HM_DOUBLE_FAULT;
5772 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
5773 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
5774
5775 break;
5776 }
5777
5778 case VMXREFLECTXCPT_TF:
5779 {
5780 rc = VINF_EM_RESET;
5781 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
5782 uExitVector));
5783 break;
5784 }
5785
5786 default:
5787 Assert(rc == VINF_SUCCESS);
5788 break;
5789 }
5790 }
5791 Assert(rc == VINF_SUCCESS || rc == VINF_HM_DOUBLE_FAULT || rc == VINF_EM_RESET);
5792 return rc;
5793}
5794
5795
5796/**
5797 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
5798 *
5799 * @returns VBox status code.
5800 * @param pVCpu Pointer to the VMCPU.
5801 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5802 * out-of-sync. Make sure to update the required fields
5803 * before using them.
5804 *
5805 * @remarks No-long-jump zone!!!
5806 */
5807static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5808{
5809 NOREF(pMixedCtx);
5810
5811 /*
5812 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
5813 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
5814 */
5815 VMMRZCallRing3Disable(pVCpu);
5816 HM_DISABLE_PREEMPT_IF_NEEDED();
5817
5818 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
5819 {
5820 uint32_t uVal = 0;
5821 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
5822 AssertRCReturn(rc, rc);
5823
5824 uint32_t uShadow = 0;
5825 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
5826 AssertRCReturn(rc, rc);
5827
5828 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
5829 CPUMSetGuestCR0(pVCpu, uVal);
5830 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
5831 }
5832
5833 HM_RESTORE_PREEMPT_IF_NEEDED();
5834 VMMRZCallRing3Enable(pVCpu);
5835 return VINF_SUCCESS;
5836}
5837
5838
5839/**
5840 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
5841 *
5842 * @returns VBox status code.
5843 * @param pVCpu Pointer to the VMCPU.
5844 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5845 * out-of-sync. Make sure to update the required fields
5846 * before using them.
5847 *
5848 * @remarks No-long-jump zone!!!
5849 */
5850static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5851{
5852 NOREF(pMixedCtx);
5853
5854 int rc = VINF_SUCCESS;
5855 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
5856 {
5857 uint32_t uVal = 0;
5858 uint32_t uShadow = 0;
5859 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
5860 AssertRCReturn(rc, rc);
5861 rc = VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
5862 AssertRCReturn(rc, rc);
5863
5864 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
5865 CPUMSetGuestCR4(pVCpu, uVal);
5866 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
5867 }
5868 return rc;
5869}
5870
5871
5872/**
5873 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
5874 *
5875 * @returns VBox status code.
5876 * @param pVCpu Pointer to the VMCPU.
5877 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5878 * out-of-sync. Make sure to update the required fields
5879 * before using them.
5880 *
5881 * @remarks No-long-jump zone!!!
5882 */
5883static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5884{
5885 int rc = VINF_SUCCESS;
5886 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
5887 {
5888 uint64_t u64Val = 0;
5889 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
5890 AssertRCReturn(rc, rc);
5891
5892 pMixedCtx->rip = u64Val;
5893 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
5894 }
5895 return rc;
5896}
5897
5898
5899/**
5900 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
5901 *
5902 * @returns VBox status code.
5903 * @param pVCpu Pointer to the VMCPU.
5904 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5905 * out-of-sync. Make sure to update the required fields
5906 * before using them.
5907 *
5908 * @remarks No-long-jump zone!!!
5909 */
5910static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5911{
5912 int rc = VINF_SUCCESS;
5913 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
5914 {
5915 uint64_t u64Val = 0;
5916 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
5917 AssertRCReturn(rc, rc);
5918
5919 pMixedCtx->rsp = u64Val;
5920 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
5921 }
5922 return rc;
5923}
5924
5925
5926/**
5927 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
5928 *
5929 * @returns VBox status code.
5930 * @param pVCpu Pointer to the VMCPU.
5931 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5932 * out-of-sync. Make sure to update the required fields
5933 * before using them.
5934 *
5935 * @remarks No-long-jump zone!!!
5936 */
5937static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5938{
5939 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
5940 {
5941 uint32_t uVal = 0;
5942 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
5943 AssertRCReturn(rc, rc);
5944
5945 pMixedCtx->eflags.u32 = uVal;
5946 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
5947 {
5948 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
5949 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
5950
5951 pMixedCtx->eflags.Bits.u1VM = 0;
5952 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
5953 }
5954
5955 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
5956 }
5957 return VINF_SUCCESS;
5958}
5959
5960
5961/**
5962 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
5963 * guest-CPU context.
5964 */
5965DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5966{
5967 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5968 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
5969 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
5970 return rc;
5971}
5972
5973
5974/**
5975 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
5976 * from the guest-state area in the VMCS.
5977 *
5978 * @param pVCpu Pointer to the VMCPU.
5979 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
5980 * out-of-sync. Make sure to update the required fields
5981 * before using them.
5982 *
5983 * @remarks No-long-jump zone!!!
5984 */
5985static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5986{
5987 uint32_t uIntrState = 0;
5988 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
5989 AssertRC(rc);
5990
5991 if (!uIntrState)
5992 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
5993 else
5994 {
5995 Assert( uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
5996 || uIntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
5997 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
5998 AssertRC(rc);
5999 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6000 AssertRC(rc);
6001
6002 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6003 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6004 }
6005}
6006
6007
6008/**
6009 * Saves the guest's activity state.
6010 *
6011 * @returns VBox status code.
6012 * @param pVCpu Pointer to the VMCPU.
6013 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6014 * out-of-sync. Make sure to update the required fields
6015 * before using them.
6016 *
6017 * @remarks No-long-jump zone!!!
6018 */
6019static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6020{
6021 NOREF(pMixedCtx);
6022 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6023 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6024 return VINF_SUCCESS;
6025}
6026
6027
6028/**
6029 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6030 * the current VMCS into the guest-CPU context.
6031 *
6032 * @returns VBox status code.
6033 * @param pVCpu Pointer to the VMCPU.
6034 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6035 * out-of-sync. Make sure to update the required fields
6036 * before using them.
6037 *
6038 * @remarks No-long-jump zone!!!
6039 */
6040static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6041{
6042 int rc = VINF_SUCCESS;
6043 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6044 {
6045 uint32_t u32Val = 0;
6046 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6047 pMixedCtx->SysEnter.cs = u32Val;
6048 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6049 }
6050
6051 uint64_t u64Val = 0;
6052 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6053 {
6054 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6055 pMixedCtx->SysEnter.eip = u64Val;
6056 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6057 }
6058 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6059 {
6060 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6061 pMixedCtx->SysEnter.esp = u64Val;
6062 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6063 }
6064 return rc;
6065}
6066
6067
6068/**
6069 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6070 * the CPU back into the guest-CPU context.
6071 *
6072 * @returns VBox status code.
6073 * @param pVCpu Pointer to the VMCPU.
6074 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6075 * out-of-sync. Make sure to update the required fields
6076 * before using them.
6077 *
6078 * @remarks No-long-jump zone!!!
6079 */
6080static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6081{
6082#if HC_ARCH_BITS == 64
6083 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
6084 {
6085 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6086 VMMRZCallRing3Disable(pVCpu);
6087 HM_DISABLE_PREEMPT_IF_NEEDED();
6088
6089 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6090 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6091 {
6092 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6093 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6094 }
6095
6096 HM_RESTORE_PREEMPT_IF_NEEDED();
6097 VMMRZCallRing3Enable(pVCpu);
6098 }
6099 else
6100 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6101#else
6102 NOREF(pMixedCtx);
6103 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6104#endif
6105
6106 return VINF_SUCCESS;
6107}
6108
6109
6110/**
6111 * Saves the auto load/store'd guest MSRs from the current VMCS into
6112 * the guest-CPU context.
6113 *
6114 * @returns VBox status code.
6115 * @param pVCpu Pointer to the VMCPU.
6116 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6117 * out-of-sync. Make sure to update the required fields
6118 * before using them.
6119 *
6120 * @remarks No-long-jump zone!!!
6121 */
6122static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6123{
6124 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6125 return VINF_SUCCESS;
6126
6127 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6128 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6129 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6130 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6131 {
6132 switch (pMsr->u32Msr)
6133 {
6134 case MSR_K8_TSC_AUX: CPUMR0SetGuestTscAux(pVCpu, pMsr->u64Value); break;
6135 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6136 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6137 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6138 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6139 default:
6140 {
6141 AssertFailed();
6142 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6143 }
6144 }
6145 }
6146
6147 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6148 return VINF_SUCCESS;
6149}
6150
6151
6152/**
6153 * Saves the guest control registers from the current VMCS into the guest-CPU
6154 * context.
6155 *
6156 * @returns VBox status code.
6157 * @param pVCpu Pointer to the VMCPU.
6158 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6159 * out-of-sync. Make sure to update the required fields
6160 * before using them.
6161 *
6162 * @remarks No-long-jump zone!!!
6163 */
6164static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6165{
6166 /* Guest CR0. Guest FPU. */
6167 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6168 AssertRCReturn(rc, rc);
6169
6170 /* Guest CR4. */
6171 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6172 AssertRCReturn(rc, rc);
6173
6174 /* Guest CR2 - updated always during the world-switch or in #PF. */
6175 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6176 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6177 {
6178 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6179 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6180
6181 PVM pVM = pVCpu->CTX_SUFF(pVM);
6182 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6183 || ( pVM->hm.s.fNestedPaging
6184 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6185 {
6186 uint64_t u64Val = 0;
6187 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6188 if (pMixedCtx->cr3 != u64Val)
6189 {
6190 CPUMSetGuestCR3(pVCpu, u64Val);
6191 if (VMMRZCallRing3IsEnabled(pVCpu))
6192 {
6193 PGMUpdateCR3(pVCpu, u64Val);
6194 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6195 }
6196 else
6197 {
6198 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6199 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6200 }
6201 }
6202
6203 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6204 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6205 {
6206 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u); AssertRCReturn(rc, rc);
6207 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u); AssertRCReturn(rc, rc);
6208 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u); AssertRCReturn(rc, rc);
6209 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u); AssertRCReturn(rc, rc);
6210
6211 if (VMMRZCallRing3IsEnabled(pVCpu))
6212 {
6213 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6214 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6215 }
6216 else
6217 {
6218 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6219 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6220 }
6221 }
6222 }
6223
6224 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6225 }
6226
6227 /*
6228 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6229 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6230 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6231 *
6232 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6233 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6234 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6235 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6236 *
6237 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6238 */
6239 if (VMMRZCallRing3IsEnabled(pVCpu))
6240 {
6241 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6242 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6243
6244 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6245 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6246
6247 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6248 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6249 }
6250
6251 return rc;
6252}
6253
6254
6255/**
6256 * Reads a guest segment register from the current VMCS into the guest-CPU
6257 * context.
6258 *
6259 * @returns VBox status code.
6260 * @param pVCpu Pointer to the VMCPU.
6261 * @param idxSel Index of the selector in the VMCS.
6262 * @param idxLimit Index of the segment limit in the VMCS.
6263 * @param idxBase Index of the segment base in the VMCS.
6264 * @param idxAccess Index of the access rights of the segment in the VMCS.
6265 * @param pSelReg Pointer to the segment selector.
6266 *
6267 * @remarks No-long-jump zone!!!
6268 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6269 * macro as that takes care of whether to read from the VMCS cache or
6270 * not.
6271 */
6272DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6273 PCPUMSELREG pSelReg)
6274{
6275 NOREF(pVCpu);
6276
6277 uint32_t u32Val = 0;
6278 int rc = VMXReadVmcs32(idxSel, &u32Val);
6279 AssertRCReturn(rc, rc);
6280 pSelReg->Sel = (uint16_t)u32Val;
6281 pSelReg->ValidSel = (uint16_t)u32Val;
6282 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6283
6284 rc = VMXReadVmcs32(idxLimit, &u32Val);
6285 AssertRCReturn(rc, rc);
6286 pSelReg->u32Limit = u32Val;
6287
6288 uint64_t u64Val = 0;
6289 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6290 AssertRCReturn(rc, rc);
6291 pSelReg->u64Base = u64Val;
6292
6293 rc = VMXReadVmcs32(idxAccess, &u32Val);
6294 AssertRCReturn(rc, rc);
6295 pSelReg->Attr.u = u32Val;
6296
6297 /*
6298 * If VT-x marks the segment as unusable, most other bits remain undefined:
6299 * - For CS the L, D and G bits have meaning.
6300 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6301 * - For the remaining data segments no bits are defined.
6302 *
6303 * The present bit and the unusable bit has been observed to be set at the
6304 * same time (the selector was supposed to invalid as we started executing
6305 * a V8086 interrupt in ring-0).
6306 *
6307 * What should be important for the rest of the VBox code, is that the P bit is
6308 * cleared. Some of the other VBox code recognizes the unusable bit, but
6309 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6310 * safe side here, we'll strip off P and other bits we don't care about. If
6311 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6312 *
6313 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6314 */
6315 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6316 {
6317 Assert(idxSel != VMX_VMCS16_GUEST_FIELD_TR); /* TR is the only selector that can never be unusable. */
6318
6319 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6320 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6321 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6322
6323 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6324#ifdef DEBUG_bird
6325 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6326 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6327 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6328#endif
6329 }
6330 return VINF_SUCCESS;
6331}
6332
6333
6334#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6335# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6336 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6337 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6338#else
6339# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6340 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_FIELD_##Sel, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6341 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6342#endif
6343
6344
6345/**
6346 * Saves the guest segment registers from the current VMCS into the guest-CPU
6347 * context.
6348 *
6349 * @returns VBox status code.
6350 * @param pVCpu Pointer to the VMCPU.
6351 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6352 * out-of-sync. Make sure to update the required fields
6353 * before using them.
6354 *
6355 * @remarks No-long-jump zone!!!
6356 */
6357static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6358{
6359 /* Guest segment registers. */
6360 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6361 {
6362 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); AssertRCReturn(rc, rc);
6363 rc = VMXLOCAL_READ_SEG(CS, cs); AssertRCReturn(rc, rc);
6364 rc = VMXLOCAL_READ_SEG(SS, ss); AssertRCReturn(rc, rc);
6365 rc = VMXLOCAL_READ_SEG(DS, ds); AssertRCReturn(rc, rc);
6366 rc = VMXLOCAL_READ_SEG(ES, es); AssertRCReturn(rc, rc);
6367 rc = VMXLOCAL_READ_SEG(FS, fs); AssertRCReturn(rc, rc);
6368 rc = VMXLOCAL_READ_SEG(GS, gs); AssertRCReturn(rc, rc);
6369
6370 /* Restore segment attributes for real-on-v86 mode hack. */
6371 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6372 {
6373 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6374 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6375 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6376 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6377 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6378 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6379 }
6380 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6381 }
6382
6383 return VINF_SUCCESS;
6384}
6385
6386
6387/**
6388 * Saves the guest descriptor table registers and task register from the current
6389 * VMCS into the guest-CPU context.
6390 *
6391 * @returns VBox status code.
6392 * @param pVCpu Pointer to the VMCPU.
6393 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6394 * out-of-sync. Make sure to update the required fields
6395 * before using them.
6396 *
6397 * @remarks No-long-jump zone!!!
6398 */
6399static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6400{
6401 int rc = VINF_SUCCESS;
6402
6403 /* Guest LDTR. */
6404 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6405 {
6406 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6407 AssertRCReturn(rc, rc);
6408 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6409 }
6410
6411 /* Guest GDTR. */
6412 uint64_t u64Val = 0;
6413 uint32_t u32Val = 0;
6414 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6415 {
6416 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6417 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6418 pMixedCtx->gdtr.pGdt = u64Val;
6419 pMixedCtx->gdtr.cbGdt = u32Val;
6420 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6421 }
6422
6423 /* Guest IDTR. */
6424 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6425 {
6426 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val); AssertRCReturn(rc, rc);
6427 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6428 pMixedCtx->idtr.pIdt = u64Val;
6429 pMixedCtx->idtr.cbIdt = u32Val;
6430 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6431 }
6432
6433 /* Guest TR. */
6434 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6435 {
6436 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6437 AssertRCReturn(rc, rc);
6438
6439 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6440 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6441 {
6442 rc = VMXLOCAL_READ_SEG(TR, tr);
6443 AssertRCReturn(rc, rc);
6444 }
6445 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6446 }
6447 return rc;
6448}
6449
6450#undef VMXLOCAL_READ_SEG
6451
6452
6453/**
6454 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6455 * context.
6456 *
6457 * @returns VBox status code.
6458 * @param pVCpu Pointer to the VMCPU.
6459 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6460 * out-of-sync. Make sure to update the required fields
6461 * before using them.
6462 *
6463 * @remarks No-long-jump zone!!!
6464 */
6465static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6466{
6467 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG))
6468 {
6469 if (!pVCpu->hm.s.fUsingHyperDR7)
6470 {
6471 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6472 uint32_t u32Val;
6473 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6474 pMixedCtx->dr[7] = u32Val;
6475 }
6476
6477 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DEBUG);
6478 }
6479 return VINF_SUCCESS;
6480}
6481
6482
6483/**
6484 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6485 *
6486 * @returns VBox status code.
6487 * @param pVCpu Pointer to the VMCPU.
6488 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6489 * out-of-sync. Make sure to update the required fields
6490 * before using them.
6491 *
6492 * @remarks No-long-jump zone!!!
6493 */
6494static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6495{
6496 NOREF(pMixedCtx);
6497
6498 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6499 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6500 return VINF_SUCCESS;
6501}
6502
6503
6504/**
6505 * Saves the entire guest state from the currently active VMCS into the
6506 * guest-CPU context. This essentially VMREADs all guest-data.
6507 *
6508 * @returns VBox status code.
6509 * @param pVCpu Pointer to the VMCPU.
6510 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6511 * out-of-sync. Make sure to update the required fields
6512 * before using them.
6513 */
6514static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6515{
6516 Assert(pVCpu);
6517 Assert(pMixedCtx);
6518
6519 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
6520 return VINF_SUCCESS;
6521
6522 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
6523 again on the ring-3 callback path, there is no real need to. */
6524 if (VMMRZCallRing3IsEnabled(pVCpu))
6525 VMMR0LogFlushDisable(pVCpu);
6526 else
6527 Assert(VMMR0IsLogFlushDisabled(pVCpu));
6528 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
6529
6530 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
6531 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6532
6533 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6534 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6535
6536 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
6537 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6538
6539 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
6540 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6541
6542 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
6543 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6544
6545 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
6546 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6547
6548 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6549 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6550
6551 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
6552 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6553
6554 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
6555 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6556
6557 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
6558 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
6559
6560 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
6561 ("Missed guest state bits while saving state; residue %RX32\n", HMVMXCPU_GST_VALUE(pVCpu)));
6562
6563 if (VMMRZCallRing3IsEnabled(pVCpu))
6564 VMMR0LogFlushEnable(pVCpu);
6565
6566 return rc;
6567}
6568
6569
6570/**
6571 * Check per-VM and per-VCPU force flag actions that require us to go back to
6572 * ring-3 for one reason or another.
6573 *
6574 * @returns VBox status code (information status code included).
6575 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6576 * ring-3.
6577 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6578 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6579 * interrupts)
6580 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6581 * all EMTs to be in ring-3.
6582 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6583 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6584 * to the EM loop.
6585 *
6586 * @param pVM Pointer to the VM.
6587 * @param pVCpu Pointer to the VMCPU.
6588 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6589 * out-of-sync. Make sure to update the required fields
6590 * before using them.
6591 */
6592static int hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6593{
6594 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6595
6596 if ( VM_FF_IS_PENDING(pVM, !pVCpu->hm.s.fSingleInstruction
6597 ? VM_FF_HP_R0_PRE_HM_MASK : VM_FF_HP_R0_PRE_HM_STEP_MASK)
6598 || VMCPU_FF_IS_PENDING(pVCpu, !pVCpu->hm.s.fSingleInstruction
6599 ? VMCPU_FF_HP_R0_PRE_HM_MASK : VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6600 {
6601 /* We need the control registers now, make sure the guest-CPU context is updated. */
6602 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
6603 AssertRCReturn(rc3, rc3);
6604
6605 /* Pending HM CR3 sync. */
6606 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6607 {
6608 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
6609 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
6610 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
6611 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6612 }
6613
6614 /* Pending HM PAE PDPEs. */
6615 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6616 {
6617 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6618 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6619 }
6620
6621 /* Pending PGM C3 sync. */
6622 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6623 {
6624 int rc2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6625 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6626 if (rc2 != VINF_SUCCESS)
6627 {
6628 AssertRC(rc2);
6629 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", rc2));
6630 return rc2;
6631 }
6632 }
6633
6634 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6635 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6636 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6637 {
6638 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6639 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6640 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6641 return rc2;
6642 }
6643
6644 /* Pending VM request packets, such as hardware interrupts. */
6645 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6646 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6647 {
6648 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
6649 return VINF_EM_PENDING_REQUEST;
6650 }
6651
6652 /* Pending PGM pool flushes. */
6653 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6654 {
6655 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
6656 return VINF_PGM_POOL_FLUSH_PENDING;
6657 }
6658
6659 /* Pending DMA requests. */
6660 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6661 {
6662 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
6663 return VINF_EM_RAW_TO_R3;
6664 }
6665 }
6666
6667 return VINF_SUCCESS;
6668}
6669
6670
6671/**
6672 * Converts any TRPM trap into a pending HM event. This is typically used when
6673 * entering from ring-3 (not longjmp returns).
6674 *
6675 * @param pVCpu Pointer to the VMCPU.
6676 */
6677static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6678{
6679 Assert(TRPMHasTrap(pVCpu));
6680 Assert(!pVCpu->hm.s.Event.fPending);
6681
6682 uint8_t uVector;
6683 TRPMEVENT enmTrpmEvent;
6684 RTGCUINT uErrCode;
6685 RTGCUINTPTR GCPtrFaultAddress;
6686 uint8_t cbInstr;
6687
6688 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6689 AssertRC(rc);
6690
6691 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6692 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6693 if (enmTrpmEvent == TRPM_TRAP)
6694 {
6695 switch (uVector)
6696 {
6697 case X86_XCPT_BP:
6698 case X86_XCPT_OF:
6699 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6700 break;
6701
6702 case X86_XCPT_PF:
6703 case X86_XCPT_DF:
6704 case X86_XCPT_TS:
6705 case X86_XCPT_NP:
6706 case X86_XCPT_SS:
6707 case X86_XCPT_GP:
6708 case X86_XCPT_AC:
6709 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6710 /* no break! */
6711 default:
6712 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6713 break;
6714 }
6715 }
6716 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6717 {
6718 if (uVector == X86_XCPT_NMI)
6719 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6720 else
6721 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6722 }
6723 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6724 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6725 else
6726 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6727
6728 rc = TRPMResetTrap(pVCpu);
6729 AssertRC(rc);
6730 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6731 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6732
6733 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6734 STAM_COUNTER_DEC(&pVCpu->hm.s.StatInjectPendingReflect);
6735}
6736
6737
6738/**
6739 * Converts any pending HM event into a TRPM trap. Typically used when leaving
6740 * VT-x to execute any instruction.
6741 *
6742 * @param pvCpu Pointer to the VMCPU.
6743 */
6744static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6745{
6746 Assert(pVCpu->hm.s.Event.fPending);
6747
6748 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6749 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6750 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6751 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6752
6753 /* If a trap was already pending, we did something wrong! */
6754 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6755
6756 TRPMEVENT enmTrapType;
6757 switch (uVectorType)
6758 {
6759 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6760 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6761 enmTrapType = TRPM_HARDWARE_INT;
6762 break;
6763
6764 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6765 enmTrapType = TRPM_SOFTWARE_INT;
6766 break;
6767
6768 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6769 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6770 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6771 enmTrapType = TRPM_TRAP;
6772 break;
6773
6774 default:
6775 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6776 enmTrapType = TRPM_32BIT_HACK;
6777 break;
6778 }
6779
6780 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6781
6782 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6783 AssertRC(rc);
6784
6785 if (fErrorCodeValid)
6786 TRPMSetErrorCode(pVCpu, uErrorCode);
6787
6788 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6789 && uVector == X86_XCPT_PF)
6790 {
6791 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6792 }
6793 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6794 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6795 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6796 {
6797 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6798 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6799 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6800 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6801 }
6802 pVCpu->hm.s.Event.fPending = false;
6803}
6804
6805
6806/**
6807 * Does the necessary state syncing before returning to ring-3 for any reason
6808 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6809 *
6810 * @returns VBox status code.
6811 * @param pVM Pointer to the VM.
6812 * @param pVCpu Pointer to the VMCPU.
6813 * @param pMixedCtx Pointer to the guest-CPU context. The data may
6814 * be out-of-sync. Make sure to update the required
6815 * fields before using them.
6816 * @param fSaveGuestState Whether to save the guest state or not.
6817 *
6818 * @remarks No-long-jmp zone!!!
6819 */
6820static int hmR0VmxLeave(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
6821{
6822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6823 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6824
6825 RTCPUID idCpu = RTMpCpuId();
6826 Log4Func(("HostCpuId=%u\n", idCpu));
6827
6828 /*
6829 * !!! IMPORTANT !!!
6830 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6831 */
6832
6833 /* Save the guest state if necessary. */
6834 if ( fSaveGuestState
6835 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
6836 {
6837 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
6838 AssertRCReturn(rc, rc);
6839 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
6840 }
6841
6842 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
6843 if (CPUMIsGuestFPUStateActive(pVCpu))
6844 {
6845 /* We shouldn't reload CR0 without saving it first. */
6846 if (!fSaveGuestState)
6847 {
6848 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6849 AssertRCReturn(rc, rc);
6850 }
6851 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
6852 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6853 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
6854 }
6855
6856 /* Restore host debug registers if necessary and resync on next R0 reentry. */
6857#ifdef VBOX_STRICT
6858 if (CPUMIsHyperDebugStateActive(pVCpu))
6859 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6860#endif
6861 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
6862 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
6863 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6864 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6865
6866#if HC_ARCH_BITS == 64
6867 /* Restore host-state bits that VT-x only restores partially. */
6868 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6869 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6870 {
6871 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6872 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6873 }
6874 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6875#endif
6876
6877#if HC_ARCH_BITS == 64
6878 /* Restore the host MSRs as we're leaving VT-x context. */
6879 if ( pVM->hm.s.fAllow64BitGuests
6880 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
6881 {
6882 /* We shouldn't reload the guest MSRs without saving it first. */
6883 if (!fSaveGuestState)
6884 {
6885 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
6886 AssertRCReturn(rc, rc);
6887 }
6888 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
6889 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6890 Assert(!pVCpu->hm.s.vmx.fRestoreHostMsrs);
6891 }
6892#endif
6893
6894 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6895 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6896
6897 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6898 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
6899 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
6900 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
6901 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6902 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6903 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6904 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6905
6906 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6907
6908 /** @todo This partially defeats the purpose of having preemption hooks.
6909 * The problem is, deregistering the hooks should be moved to a place that
6910 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6911 * context.
6912 */
6913 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6914 {
6915 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6916 AssertRCReturn(rc, rc);
6917
6918 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6919 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6920 }
6921 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6922 NOREF(idCpu);
6923
6924 return VINF_SUCCESS;
6925}
6926
6927
6928/**
6929 * Leaves the VT-x session.
6930 *
6931 * @returns VBox status code.
6932 * @param pVM Pointer to the VM.
6933 * @param pVCpu Pointer to the VMCPU.
6934 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6935 * out-of-sync. Make sure to update the required fields
6936 * before using them.
6937 *
6938 * @remarks No-long-jmp zone!!!
6939 */
6940DECLINLINE(int) hmR0VmxLeaveSession(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6941{
6942 HM_DISABLE_PREEMPT_IF_NEEDED();
6943 HMVMX_ASSERT_CPU_SAFE();
6944 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6945 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6946
6947 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6948 and done this from the VMXR0ThreadCtxCallback(). */
6949 if (!pVCpu->hm.s.fLeaveDone)
6950 {
6951 int rc2 = hmR0VmxLeave(pVM, pVCpu, pMixedCtx, true /* fSaveGuestState */);
6952 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT_IF_NEEDED(), rc2);
6953 pVCpu->hm.s.fLeaveDone = true;
6954 }
6955
6956 /*
6957 * !!! IMPORTANT !!!
6958 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6959 */
6960
6961 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6962 /** @todo This is bad. Deregistering here means we need to VMCLEAR always
6963 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6964 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
6965 VMMR0ThreadCtxHooksDeregister(pVCpu);
6966
6967 /* Leave HM context. This takes care of local init (term). */
6968 int rc = HMR0LeaveCpu(pVCpu);
6969
6970 HM_RESTORE_PREEMPT_IF_NEEDED();
6971
6972 return rc;
6973}
6974
6975
6976/**
6977 * Does the necessary state syncing before doing a longjmp to ring-3.
6978 *
6979 * @returns VBox status code.
6980 * @param pVM Pointer to the VM.
6981 * @param pVCpu Pointer to the VMCPU.
6982 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6983 * out-of-sync. Make sure to update the required fields
6984 * before using them.
6985 *
6986 * @remarks No-long-jmp zone!!!
6987 */
6988DECLINLINE(int) hmR0VmxLongJmpToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6989{
6990 return hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
6991}
6992
6993
6994/**
6995 * Take necessary actions before going back to ring-3.
6996 *
6997 * An action requires us to go back to ring-3. This function does the necessary
6998 * steps before we can safely return to ring-3. This is not the same as longjmps
6999 * to ring-3, this is voluntary and prepares the guest so it may continue
7000 * executing outside HM (recompiler/IEM).
7001 *
7002 * @returns VBox status code.
7003 * @param pVM Pointer to the VM.
7004 * @param pVCpu Pointer to the VMCPU.
7005 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7006 * out-of-sync. Make sure to update the required fields
7007 * before using them.
7008 * @param rcExit The reason for exiting to ring-3. Can be
7009 * VINF_VMM_UNKNOWN_RING3_CALL.
7010 */
7011static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, int rcExit)
7012{
7013 Assert(pVM);
7014 Assert(pVCpu);
7015 Assert(pMixedCtx);
7016 HMVMX_ASSERT_PREEMPT_SAFE();
7017
7018 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7019 {
7020 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7021 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7022 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7023 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7024 }
7025
7026 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7027 VMMRZCallRing3Disable(pVCpu);
7028 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, rcExit));
7029
7030 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7031 if (pVCpu->hm.s.Event.fPending)
7032 {
7033 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7034 Assert(!pVCpu->hm.s.Event.fPending);
7035 }
7036
7037 /* Save guest state and restore host state bits. */
7038 int rc = hmR0VmxLeaveSession(pVM, pVCpu, pMixedCtx);
7039 AssertRCReturn(rc, rc);
7040 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7041
7042 /* Sync recompiler state. */
7043 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7044 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7045 | CPUM_CHANGED_LDTR
7046 | CPUM_CHANGED_GDTR
7047 | CPUM_CHANGED_IDTR
7048 | CPUM_CHANGED_TR
7049 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7050 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7051 if ( pVM->hm.s.fNestedPaging
7052 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7053 {
7054 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7055 }
7056
7057 Assert(!pVCpu->hm.s.fClearTrapFlag);
7058
7059 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7060 if (rcExit != VINF_EM_RAW_INTERRUPT)
7061 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7062
7063 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7064
7065 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7066 VMMRZCallRing3RemoveNotification(pVCpu);
7067 VMMRZCallRing3Enable(pVCpu);
7068
7069 return rc;
7070}
7071
7072
7073/**
7074 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7075 * longjump to ring-3 and possibly get preempted.
7076 *
7077 * @returns VBox status code.
7078 * @param pVCpu Pointer to the VMCPU.
7079 * @param enmOperation The operation causing the ring-3 longjump.
7080 * @param pvUser Opaque pointer to the guest-CPU context. The data
7081 * may be out-of-sync. Make sure to update the required
7082 * fields before using them.
7083 */
7084DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7085{
7086 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7087 {
7088 /*
7089 * !!! IMPORTANT !!!
7090 * If you modify code here, make sure to check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs
7091 * to be updated too. This is a stripped down version which gets out ASAP trying to not trigger any assertion.
7092 */
7093 VMMRZCallRing3RemoveNotification(pVCpu);
7094 VMMRZCallRing3Disable(pVCpu);
7095 HM_DISABLE_PREEMPT_IF_NEEDED();
7096
7097 PVM pVM = pVCpu->CTX_SUFF(pVM);
7098 if (CPUMIsGuestFPUStateActive(pVCpu))
7099 CPUMR0SaveGuestFPU(pVM, pVCpu, (PCPUMCTX)pvUser);
7100
7101 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7102
7103#if HC_ARCH_BITS == 64
7104 /* Restore host-state bits that VT-x only restores partially. */
7105 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7106 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7107 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7108 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7109
7110 /* Restore the host MSRs as we're leaving VT-x context. */
7111 if ( pVM->hm.s.fAllow64BitGuests
7112 && pVCpu->hm.s.vmx.fRestoreHostMsrs)
7113 {
7114 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7115 }
7116#endif
7117 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7118 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7119 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7120 {
7121 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7122 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7123 }
7124
7125 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
7126 VMMR0ThreadCtxHooksDeregister(pVCpu);
7127
7128 HMR0LeaveCpu(pVCpu);
7129 HM_RESTORE_PREEMPT_IF_NEEDED();
7130 return VINF_SUCCESS;
7131 }
7132
7133 Assert(pVCpu);
7134 Assert(pvUser);
7135 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7136 HMVMX_ASSERT_PREEMPT_SAFE();
7137
7138 VMMRZCallRing3Disable(pVCpu);
7139 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7140
7141 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32\n enmOperation=%d", pVCpu, pVCpu->idCpu,
7142 enmOperation));
7143
7144 int rc = hmR0VmxLongJmpToRing3(pVCpu->CTX_SUFF(pVM), pVCpu, (PCPUMCTX)pvUser);
7145 AssertRCReturn(rc, rc);
7146
7147 VMMRZCallRing3Enable(pVCpu);
7148 return VINF_SUCCESS;
7149}
7150
7151
7152/**
7153 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7154 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7155 *
7156 * @param pVCpu Pointer to the VMCPU.
7157 */
7158DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7159{
7160 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7161 {
7162 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7163 {
7164 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7165 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7166 AssertRC(rc);
7167 Log4(("Setup interrupt-window exiting\n"));
7168 }
7169 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7170}
7171
7172
7173/**
7174 * Clears the interrupt-window exiting control in the VMCS.
7175 *
7176 * @param pVCpu Pointer to the VMCPU.
7177 */
7178DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7179{
7180 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7181 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7182 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7183 AssertRC(rc);
7184 Log4(("Cleared interrupt-window exiting\n"));
7185}
7186
7187
7188/**
7189 * Evaluates the event to be delivered to the guest and sets it as the pending
7190 * event.
7191 *
7192 * @param pVCpu Pointer to the VMCPU.
7193 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7194 * out-of-sync. Make sure to update the required fields
7195 * before using them.
7196 */
7197static void hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7198{
7199 Assert(!pVCpu->hm.s.Event.fPending);
7200
7201 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7202 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7203 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7204 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7205
7206 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7207 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7208 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7209 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7210 Assert(!TRPMHasTrap(pVCpu));
7211
7212 /** @todo SMI. SMIs take priority over NMIs. */
7213 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts . */
7214 {
7215 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7216 if ( !fBlockMovSS
7217 && !fBlockSti)
7218 {
7219 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7220 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7221 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7222 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7223
7224 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddres */);
7225 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7226 }
7227 else
7228 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7229 }
7230 /*
7231 * Check if the guest can receive external interrupts (PIC/APIC). Once we do PDMGetInterrupt() we -must- deliver
7232 * the interrupt ASAP. We must not execute any guest code until we inject the interrupt.
7233 */
7234 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7235 && !pVCpu->hm.s.fSingleInstruction)
7236 {
7237 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7238 AssertRC(rc);
7239 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7240 if ( !fBlockInt
7241 && !fBlockSti
7242 && !fBlockMovSS)
7243 {
7244 uint8_t u8Interrupt;
7245 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7246 if (RT_SUCCESS(rc))
7247 {
7248 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7249 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7250 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7251
7252 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7253 }
7254 else
7255 {
7256 /** @todo Does this actually happen? If not turn it into an assertion. */
7257 Assert(!VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC)));
7258 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7259 }
7260 }
7261 else
7262 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7263 }
7264}
7265
7266
7267/**
7268 * Sets a pending-debug exception to be delivered to the guest if the guest is
7269 * single-stepping.
7270 *
7271 * @param pVCpu Pointer to the VMCPU.
7272 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7273 * out-of-sync. Make sure to update the required fields
7274 * before using them.
7275 */
7276DECLINLINE(void) hmR0VmxSetPendingDebugXcpt(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7277{
7278 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7279 if (pMixedCtx->eflags.Bits.u1TF) /* We don't have any IA32_DEBUGCTL MSR for guests. Treat as all bits 0. */
7280 {
7281 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7282 AssertRC(rc);
7283 }
7284}
7285
7286
7287/**
7288 * Injects any pending events into the guest if the guest is in a state to
7289 * receive them.
7290 *
7291 * @returns VBox status code (informational status codes included).
7292 * @param pVCpu Pointer to the VMCPU.
7293 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7294 * out-of-sync. Make sure to update the required fields
7295 * before using them.
7296 */
7297static int hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7298{
7299 HMVMX_ASSERT_PREEMPT_SAFE();
7300 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7301
7302 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7303 uint32_t uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7304 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7305 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7306
7307 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7308 Assert( !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI) /* We don't support block-by-NMI and SMI yet.*/
7309 && !(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI));
7310 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7311 Assert(!TRPMHasTrap(pVCpu));
7312
7313 int rc = VINF_SUCCESS;
7314 if (pVCpu->hm.s.Event.fPending)
7315 {
7316 /*
7317 * Clear any interrupt-window exiting control if we're going to inject an interrupt. Saves one extra
7318 * VM-exit in situations where we previously setup interrupt-window exiting but got other VM-exits and
7319 * ended up enabling interrupts outside VT-x.
7320 */
7321 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7322 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7323 && ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT
7324 || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI))
7325 {
7326 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7327 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7328 }
7329#if defined(VBOX_STRICT) || defined(VBOX_WITH_STATISTICS)
7330 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7331 {
7332 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7333 AssertRCReturn(rc, rc);
7334 const bool fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7335 Assert(!fBlockInt);
7336 Assert(!fBlockSti);
7337 Assert(!fBlockMovSS);
7338 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7339 }
7340 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7341 {
7342 Assert(!fBlockSti);
7343 Assert(!fBlockMovSS);
7344 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT));
7345 }
7346#endif
7347 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7348 (uint8_t)uIntType));
7349 rc = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7350 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, &uIntrState);
7351 AssertRCReturn(rc, rc);
7352
7353 /* Update the interruptibility-state as it could have been changed by
7354 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7355 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7356 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7357
7358#ifdef VBOX_WITH_STATISTICS
7359 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7360 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7361 else
7362 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7363#endif
7364 }
7365
7366 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7367 if ( fBlockSti
7368 || fBlockMovSS)
7369 {
7370 if ( !pVCpu->hm.s.fSingleInstruction
7371 && !DBGFIsStepping(pVCpu))
7372 {
7373 /*
7374 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7375 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7376 * See Intel spec. 27.3.4 "Saving Non-Register State".
7377 */
7378 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7379 AssertRCReturn(rc2, rc2);
7380 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
7381 }
7382 else if (pMixedCtx->eflags.Bits.u1TF)
7383 {
7384 /*
7385 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7386 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7387 */
7388 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7389 uIntrState = 0;
7390 }
7391 }
7392
7393 /*
7394 * There's no need to clear the VM entry-interruption information field here if we're not injecting anything.
7395 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7396 */
7397 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
7398 AssertRC(rc2);
7399
7400 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7401 NOREF(fBlockMovSS); NOREF(fBlockSti);
7402 return rc;
7403}
7404
7405
7406/**
7407 * Sets an invalid-opcode (#UD) exception as pending-for-injection into the VM.
7408 *
7409 * @param pVCpu Pointer to the VMCPU.
7410 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7411 * out-of-sync. Make sure to update the required fields
7412 * before using them.
7413 */
7414DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7415{
7416 NOREF(pMixedCtx);
7417 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7418 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7419}
7420
7421
7422/**
7423 * Injects a double-fault (#DF) exception into the VM.
7424 *
7425 * @returns VBox status code (informational status code included).
7426 * @param pVCpu Pointer to the VMCPU.
7427 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7428 * out-of-sync. Make sure to update the required fields
7429 * before using them.
7430 */
7431DECLINLINE(int) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t *puIntrState)
7432{
7433 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7434 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7435 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7436 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
7437 puIntrState);
7438}
7439
7440
7441/**
7442 * Sets a debug (#DB) exception as pending-for-injection into the VM.
7443 *
7444 * @param pVCpu Pointer to the VMCPU.
7445 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7446 * out-of-sync. Make sure to update the required fields
7447 * before using them.
7448 */
7449DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7450{
7451 NOREF(pMixedCtx);
7452 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7453 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7454 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7455}
7456
7457
7458/**
7459 * Sets an overflow (#OF) exception as pending-for-injection into the VM.
7460 *
7461 * @param pVCpu Pointer to the VMCPU.
7462 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7463 * out-of-sync. Make sure to update the required fields
7464 * before using them.
7465 * @param cbInstr The value of RIP that is to be pushed on the guest
7466 * stack.
7467 */
7468DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7469{
7470 NOREF(pMixedCtx);
7471 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7472 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7473 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7474}
7475
7476
7477/**
7478 * Injects a general-protection (#GP) fault into the VM.
7479 *
7480 * @returns VBox status code (informational status code included).
7481 * @param pVCpu Pointer to the VMCPU.
7482 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7483 * out-of-sync. Make sure to update the required fields
7484 * before using them.
7485 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7486 * mode, i.e. in real-mode it's not valid).
7487 * @param u32ErrorCode The error code associated with the #GP.
7488 */
7489DECLINLINE(int) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7490 uint32_t *puIntrState)
7491{
7492 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7493 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7494 if (fErrorCodeValid)
7495 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7496 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
7497 puIntrState);
7498}
7499
7500
7501/**
7502 * Sets a general-protection (#GP) exception as pending-for-injection into the
7503 * VM.
7504 *
7505 * @param pVCpu Pointer to the VMCPU.
7506 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7507 * out-of-sync. Make sure to update the required fields
7508 * before using them.
7509 * @param u32ErrorCode The error code associated with the #GP.
7510 */
7511DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7512{
7513 NOREF(pMixedCtx);
7514 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7515 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7516 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7517 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7518}
7519
7520
7521/**
7522 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7523 *
7524 * @param pVCpu Pointer to the VMCPU.
7525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7526 * out-of-sync. Make sure to update the required fields
7527 * before using them.
7528 * @param uVector The software interrupt vector number.
7529 * @param cbInstr The value of RIP that is to be pushed on the guest
7530 * stack.
7531 */
7532DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7533{
7534 NOREF(pMixedCtx);
7535 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7536 if ( uVector == X86_XCPT_BP
7537 || uVector == X86_XCPT_OF)
7538 {
7539 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7540 }
7541 else
7542 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7543 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7544}
7545
7546
7547/**
7548 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7549 * stack.
7550 *
7551 * @returns VBox status code (information status code included).
7552 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7553 * @param pVM Pointer to the VM.
7554 * @param pMixedCtx Pointer to the guest-CPU context.
7555 * @param uValue The value to push to the guest stack.
7556 */
7557DECLINLINE(int) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7558{
7559 /*
7560 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7561 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7562 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7563 */
7564 if (pMixedCtx->sp == 1)
7565 return VINF_EM_RESET;
7566 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7567 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7568 AssertRCReturn(rc, rc);
7569 return rc;
7570}
7571
7572
7573/**
7574 * Injects an event into the guest upon VM-entry by updating the relevant fields
7575 * in the VM-entry area in the VMCS.
7576 *
7577 * @returns VBox status code (informational error codes included).
7578 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7579 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7580 *
7581 * @param pVCpu Pointer to the VMCPU.
7582 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7583 * be out-of-sync. Make sure to update the required
7584 * fields before using them.
7585 * @param u64IntInfo The VM-entry interruption-information field.
7586 * @param cbInstr The VM-entry instruction length in bytes (for
7587 * software interrupts, exceptions and privileged
7588 * software exceptions).
7589 * @param u32ErrCode The VM-entry exception error code.
7590 * @param GCPtrFaultAddress The page-fault address for #PF exceptions.
7591 * @param puIntrState Pointer to the current guest interruptibility-state.
7592 * This interruptibility-state will be updated if
7593 * necessary. This cannot not be NULL.
7594 *
7595 * @remarks Requires CR0!
7596 * @remarks No-long-jump zone!!!
7597 */
7598static int hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
7599 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, uint32_t *puIntrState)
7600{
7601 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7602 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
7603 Assert(puIntrState);
7604 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7605
7606 const uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7607 const uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7608
7609#ifdef VBOX_STRICT
7610 /* Validate the error-code-valid bit for hardware exceptions. */
7611 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT)
7612 {
7613 switch (uVector)
7614 {
7615 case X86_XCPT_PF:
7616 case X86_XCPT_DF:
7617 case X86_XCPT_TS:
7618 case X86_XCPT_NP:
7619 case X86_XCPT_SS:
7620 case X86_XCPT_GP:
7621 case X86_XCPT_AC:
7622 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7623 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7624 /* fallthru */
7625 default:
7626 break;
7627 }
7628 }
7629#endif
7630
7631 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7632 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7633 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7634
7635 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7636
7637 /* We require CR0 to check if the guest is in real-mode. */
7638 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7639 AssertRCReturn(rc, rc);
7640
7641 /*
7642 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
7643 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
7644 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
7645 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7646 */
7647 if (CPUMIsGuestInRealModeEx(pMixedCtx))
7648 {
7649 PVM pVM = pVCpu->CTX_SUFF(pVM);
7650 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
7651 {
7652 Assert(PDMVmmDevHeapIsEnabled(pVM));
7653 Assert(pVM->hm.s.vmx.pRealModeTSS);
7654
7655 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
7656 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7657 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7658 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7659 AssertRCReturn(rc, rc);
7660 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
7661
7662 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7663 const size_t cbIdtEntry = sizeof(X86IDTR16);
7664 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7665 {
7666 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7667 if (uVector == X86_XCPT_DF)
7668 return VINF_EM_RESET;
7669 else if (uVector == X86_XCPT_GP)
7670 {
7671 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7672 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, puIntrState);
7673 }
7674
7675 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
7676 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
7677 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, puIntrState);
7678 }
7679
7680 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7681 uint16_t uGuestIp = pMixedCtx->ip;
7682 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7683 {
7684 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7685 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7686 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7687 }
7688 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7689 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7690
7691 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7692 X86IDTR16 IdtEntry;
7693 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7694 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7695 AssertRCReturn(rc, rc);
7696
7697 /* Construct the stack frame for the interrupt/exception handler. */
7698 rc = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7699 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7700 rc |= hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7701 AssertRCReturn(rc, rc);
7702
7703 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7704 if (rc == VINF_SUCCESS)
7705 {
7706 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7707 pMixedCtx->rip = IdtEntry.offSel;
7708 pMixedCtx->cs.Sel = IdtEntry.uSel;
7709 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7710 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7711 && uVector == X86_XCPT_PF)
7712 {
7713 pMixedCtx->cr2 = GCPtrFaultAddress;
7714 }
7715
7716 /* If any other guest-state bits are changed here, make sure to update
7717 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7718 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
7719 | HM_CHANGED_GUEST_RIP
7720 | HM_CHANGED_GUEST_RFLAGS
7721 | HM_CHANGED_GUEST_RSP);
7722
7723 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7724 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7725 {
7726 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7727 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7728 Log4(("Clearing inhibition due to STI.\n"));
7729 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7730 }
7731 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x instrlen=%#x\n", u32IntInfo, u32ErrCode, cbInstr));
7732
7733 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7734 it, if we are returning to ring-3 before executing guest code. */
7735 pVCpu->hm.s.Event.fPending = false;
7736 }
7737 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RESET);
7738 return rc;
7739 }
7740 else
7741 {
7742 /*
7743 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
7744 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7745 */
7746 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7747 }
7748 }
7749
7750 /* Validate. */
7751 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7752 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK(u32IntInfo)); /* Bit 12 MBZ. */
7753 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7754
7755 /* Inject. */
7756 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7757 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7758 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7759 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7760
7761 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7762 && uVector == X86_XCPT_PF)
7763 {
7764 pMixedCtx->cr2 = GCPtrFaultAddress;
7765 }
7766
7767 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
7768 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7769
7770 AssertRCReturn(rc, rc);
7771 return rc;
7772}
7773
7774
7775/**
7776 * Clears the interrupt-window exiting control in the VMCS and if necessary
7777 * clears the current event in the VMCS as well.
7778 *
7779 * @returns VBox status code.
7780 * @param pVCpu Pointer to the VMCPU.
7781 *
7782 * @remarks Use this function only to clear events that have not yet been
7783 * delivered to the guest but are injected in the VMCS!
7784 * @remarks No-long-jump zone!!!
7785 */
7786static void hmR0VmxClearEventVmcs(PVMCPU pVCpu)
7787{
7788 int rc;
7789 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
7790
7791 /* Clear interrupt-window exiting control. */
7792 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7793 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7794
7795 if (!pVCpu->hm.s.Event.fPending)
7796 return;
7797
7798#ifdef VBOX_STRICT
7799 uint32_t u32EntryInfo;
7800 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
7801 AssertRC(rc);
7802 Assert(VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo));
7803#endif
7804
7805 /* Clear the entry-interruption field (including the valid bit). */
7806 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
7807 AssertRC(rc);
7808
7809 /* Clear the pending debug exception field. */
7810 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
7811 AssertRC(rc);
7812
7813 /* We deliberately don't clear "hm.s.Event.fPending" here, it's taken
7814 care of in hmR0VmxExitToRing3() converting the pending event to TRPM. */
7815}
7816
7817
7818/**
7819 * Enters the VT-x session.
7820 *
7821 * @returns VBox status code.
7822 * @param pVM Pointer to the VM.
7823 * @param pVCpu Pointer to the VMCPU.
7824 * @param pCpu Pointer to the CPU info struct.
7825 */
7826VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
7827{
7828 AssertPtr(pVM);
7829 AssertPtr(pVCpu);
7830 Assert(pVM->hm.s.vmx.fSupported);
7831 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7832 NOREF(pCpu); NOREF(pVM);
7833
7834 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7835 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7836
7837#ifdef VBOX_STRICT
7838 /* Make sure we're in VMX root mode. */
7839 RTCCUINTREG u32HostCR4 = ASMGetCR4();
7840 if (!(u32HostCR4 & X86_CR4_VMXE))
7841 {
7842 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
7843 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7844 }
7845#endif
7846
7847 /*
7848 * Load the VCPU's VMCS as the current (and active) one.
7849 */
7850 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7851 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7852 if (RT_FAILURE(rc))
7853 return rc;
7854
7855 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7856 pVCpu->hm.s.fLeaveDone = false;
7857 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7858
7859 return VINF_SUCCESS;
7860}
7861
7862
7863/**
7864 * The thread-context callback (only on platforms which support it).
7865 *
7866 * @param enmEvent The thread-context event.
7867 * @param pVCpu Pointer to the VMCPU.
7868 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7869 * @thread EMT(pVCpu)
7870 */
7871VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7872{
7873 NOREF(fGlobalInit);
7874
7875 switch (enmEvent)
7876 {
7877 case RTTHREADCTXEVENT_PREEMPTING:
7878 {
7879 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7880 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7881 VMCPU_ASSERT_EMT(pVCpu);
7882
7883 PVM pVM = pVCpu->CTX_SUFF(pVM);
7884 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
7885
7886 /* No longjmps (logger flushes, locks) in this fragile context. */
7887 VMMRZCallRing3Disable(pVCpu);
7888 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7889
7890 /*
7891 * Restore host-state (FPU, debug etc.)
7892 */
7893 if (!pVCpu->hm.s.fLeaveDone)
7894 {
7895 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
7896 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
7897 hmR0VmxLeave(pVM, pVCpu, pMixedCtx, false /* fSaveGuestState */);
7898 pVCpu->hm.s.fLeaveDone = true;
7899 }
7900
7901 /* Leave HM context, takes care of local init (term). */
7902 int rc = HMR0LeaveCpu(pVCpu);
7903 AssertRC(rc); NOREF(rc);
7904
7905 /* Restore longjmp state. */
7906 VMMRZCallRing3Enable(pVCpu);
7907 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptPreempting);
7908 break;
7909 }
7910
7911 case RTTHREADCTXEVENT_RESUMED:
7912 {
7913 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7914 Assert(VMMR0ThreadCtxHooksAreRegistered(pVCpu));
7915 VMCPU_ASSERT_EMT(pVCpu);
7916
7917 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7918 VMMRZCallRing3Disable(pVCpu);
7919 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7920
7921 /* Initialize the bare minimum state required for HM. This takes care of
7922 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7923 int rc = HMR0EnterCpu(pVCpu);
7924 AssertRC(rc);
7925 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
7926
7927 /* Load the active VMCS as the current one. */
7928 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7929 {
7930 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7931 AssertRC(rc); NOREF(rc);
7932 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7933 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7934 }
7935 pVCpu->hm.s.fLeaveDone = false;
7936
7937 /* Restore longjmp state. */
7938 VMMRZCallRing3Enable(pVCpu);
7939 break;
7940 }
7941
7942 default:
7943 break;
7944 }
7945}
7946
7947
7948/**
7949 * Saves the host state in the VMCS host-state.
7950 * Sets up the VM-exit MSR-load area.
7951 *
7952 * The CPU state will be loaded from these fields on every successful VM-exit.
7953 *
7954 * @returns VBox status code.
7955 * @param pVM Pointer to the VM.
7956 * @param pVCpu Pointer to the VMCPU.
7957 *
7958 * @remarks No-long-jump zone!!!
7959 */
7960static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
7961{
7962 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7963
7964 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
7965 return VINF_SUCCESS;
7966
7967 int rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
7968 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7969
7970 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
7971 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7972
7973 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
7974 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
7975
7976 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
7977 return rc;
7978}
7979
7980
7981/**
7982 * Saves the host state in the VMCS host-state.
7983 *
7984 * @returns VBox status code.
7985 * @param pVM Pointer to the VM.
7986 * @param pVCpu Pointer to the VMCPU.
7987 *
7988 * @remarks No-long-jump zone!!!
7989 */
7990VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
7991{
7992 AssertPtr(pVM);
7993 AssertPtr(pVCpu);
7994
7995 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
7996
7997 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
7998 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
7999 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8000 return hmR0VmxSaveHostState(pVM, pVCpu);
8001}
8002
8003
8004/**
8005 * Loads the guest state into the VMCS guest-state area. The CPU state will be
8006 * loaded from these fields on every successful VM-entry.
8007 *
8008 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas.
8009 * Sets up the VM-entry controls.
8010 * Sets up the appropriate VMX non-root function to execute guest code based on
8011 * the guest CPU mode.
8012 *
8013 * @returns VBox status code.
8014 * @param pVM Pointer to the VM.
8015 * @param pVCpu Pointer to the VMCPU.
8016 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8017 * out-of-sync. Make sure to update the required fields
8018 * before using them.
8019 *
8020 * @remarks No-long-jump zone!!!
8021 */
8022static int hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8023{
8024 AssertPtr(pVM);
8025 AssertPtr(pVCpu);
8026 AssertPtr(pMixedCtx);
8027 HMVMX_ASSERT_PREEMPT_SAFE();
8028
8029#ifdef LOG_ENABLED
8030 /** @todo r=ramshankar: I'm not able to use VMMRZCallRing3Disable() here,
8031 * probably not initialized yet? Anyway this will do for now.
8032 *
8033 * Update: Should be possible once VMXR0LoadGuestState() is removed as an
8034 * interface and disable ring-3 calls when thread-context hooks are not
8035 * available. */
8036 bool fCallerDisabledLogFlush = VMMR0IsLogFlushDisabled(pVCpu);
8037 VMMR0LogFlushDisable(pVCpu);
8038#endif
8039
8040 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8041
8042 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8043
8044 /* Determine real-on-v86 mode. */
8045 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8046 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8047 && CPUMIsGuestInRealModeEx(pMixedCtx))
8048 {
8049 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8050 }
8051
8052 /*
8053 * Load the guest-state into the VMCS.
8054 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8055 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8056 */
8057 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8058 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8059
8060 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8061 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8062 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8063
8064 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8065 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8066 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8067
8068 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8069 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8070
8071 rc = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8072 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestCR3AndCR4: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8073
8074 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8075 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8076 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8077
8078 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8079 determine we don't have to swap EFER after all. */
8080 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8081 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadSharedMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8082
8083 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8084 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8085
8086 /*
8087 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8088 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8089 */
8090 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8091 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8092
8093 /* Clear any unused and reserved bits. */
8094 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2);
8095
8096#ifdef LOG_ENABLED
8097 /* Only reenable log-flushing if the caller has it enabled. */
8098 if (!fCallerDisabledLogFlush)
8099 VMMR0LogFlushEnable(pVCpu);
8100#endif
8101
8102 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8103 return rc;
8104}
8105
8106
8107/**
8108 * Loads the state shared between the host and guest into the VMCS.
8109 *
8110 * @param pVM Pointer to the VM.
8111 * @param pVCpu Pointer to the VMCPU.
8112 * @param pCtx Pointer to the guest-CPU context.
8113 *
8114 * @remarks No-long-jump zone!!!
8115 */
8116static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8117{
8118 NOREF(pVM);
8119
8120 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8121 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8122
8123 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8124 {
8125 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8126 AssertRC(rc);
8127 }
8128
8129 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8130 {
8131 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8132 AssertRC(rc);
8133
8134 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8135 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8136 {
8137 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8138 AssertRC(rc);
8139 }
8140 }
8141
8142 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS))
8143 {
8144#if HC_ARCH_BITS == 64
8145 if (pVM->hm.s.fAllow64BitGuests)
8146 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8147#endif
8148 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
8149 }
8150
8151 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8152 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8153}
8154
8155
8156/**
8157 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8158 *
8159 * @param pVM Pointer to the VM.
8160 * @param pVCpu Pointer to the VMCPU.
8161 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8162 * out-of-sync. Make sure to update the required fields
8163 * before using them.
8164 */
8165DECLINLINE(void) hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8166{
8167 HMVMX_ASSERT_PREEMPT_SAFE();
8168
8169 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8170#ifdef HMVMX_SYNC_FULL_GUEST_STATE
8171 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8172#endif
8173
8174 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8175 {
8176 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8177 AssertRC(rc);
8178 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8179 }
8180 else if (HMCPU_CF_VALUE(pVCpu))
8181 {
8182 int rc = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8183 AssertRC(rc);
8184 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8185 }
8186
8187 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8188 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8189 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8190 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8191}
8192
8193
8194/**
8195 * Does the preparations before executing guest code in VT-x.
8196 *
8197 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8198 * recompiler. We must be cautious what we do here regarding committing
8199 * guest-state information into the VMCS assuming we assuredly execute the
8200 * guest in VT-x mode. If we fall back to the recompiler after updating the VMCS
8201 * and clearing the common-state (TRPM/forceflags), we must undo those changes
8202 * so that the recompiler can (and should) use them when it resumes guest
8203 * execution. Otherwise such operations must be done when we can no longer
8204 * exit to ring-3.
8205 *
8206 * @returns Strict VBox status code.
8207 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8208 * have been disabled.
8209 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8210 * double-fault into the guest.
8211 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8212 *
8213 * @param pVM Pointer to the VM.
8214 * @param pVCpu Pointer to the VMCPU.
8215 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8216 * out-of-sync. Make sure to update the required fields
8217 * before using them.
8218 * @param pVmxTransient Pointer to the VMX transient structure.
8219 */
8220static int hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8221{
8222 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8223
8224#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8225 PGMRZDynMapFlushAutoSet(pVCpu);
8226#endif
8227
8228 /* Check force flag actions that might require us to go back to ring-3. */
8229 int rc = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx);
8230 if (rc != VINF_SUCCESS)
8231 return rc;
8232
8233#ifndef IEM_VERIFICATION_MODE_FULL
8234 /* Setup the Virtualized APIC accesses. pMixedCtx->msrApicBase is always up-to-date. It's not part of the VMCS. */
8235 if ( pVCpu->hm.s.vmx.u64MsrApicBase != pMixedCtx->msrApicBase
8236 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC))
8237 {
8238 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8239 RTGCPHYS GCPhysApicBase;
8240 GCPhysApicBase = pMixedCtx->msrApicBase;
8241 GCPhysApicBase &= PAGE_BASE_GC_MASK;
8242
8243 /* Unalias any existing mapping. */
8244 rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8245 AssertRCReturn(rc, rc);
8246
8247 /* Map the HC APIC-access page into the GC space, this also updates the shadow page tables if necessary. */
8248 Log4(("Mapped HC APIC-access page into GC: GCPhysApicBase=%#RGv\n", GCPhysApicBase));
8249 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8250 AssertRCReturn(rc, rc);
8251
8252 pVCpu->hm.s.vmx.u64MsrApicBase = pMixedCtx->msrApicBase;
8253 }
8254#endif /* !IEM_VERIFICATION_MODE_FULL */
8255
8256 /* Load the guest state bits, we can handle longjmps/getting preempted here. */
8257 hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8258
8259 /*
8260 * Evaluate events as pending-for-injection into the guest. Toggling of force-flags here is safe as long as
8261 * we update TRPM on premature exits to ring-3 before executing guest code. We must NOT restore the force-flags.
8262 */
8263 if (TRPMHasTrap(pVCpu))
8264 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8265 else if (!pVCpu->hm.s.Event.fPending)
8266 hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8267
8268 /*
8269 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8270 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8271 */
8272 rc = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx);
8273 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8274 {
8275 Assert(rc == VINF_EM_RESET);
8276 return rc;
8277 }
8278
8279 /*
8280 * No longjmps to ring-3 from this point on!!!
8281 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8282 * This also disables flushing of the R0-logger instance (if any).
8283 */
8284 VMMRZCallRing3Disable(pVCpu);
8285
8286 /*
8287 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8288 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8289 *
8290 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8291 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8292 *
8293 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8294 * executing guest code.
8295 */
8296 pVmxTransient->uEflags = ASMIntDisableFlags();
8297 if ( VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8298 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8299 {
8300 hmR0VmxClearEventVmcs(pVCpu);
8301 ASMSetFlags(pVmxTransient->uEflags);
8302 VMMRZCallRing3Enable(pVCpu);
8303 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8304 return VINF_EM_RAW_TO_R3;
8305 }
8306
8307 if (RTThreadPreemptIsPending(NIL_RTTHREAD))
8308 {
8309 hmR0VmxClearEventVmcs(pVCpu);
8310 ASMSetFlags(pVmxTransient->uEflags);
8311 VMMRZCallRing3Enable(pVCpu);
8312 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8313 return VINF_EM_RAW_INTERRUPT;
8314 }
8315
8316 /* We've injected any pending events. This is really the point of no return (to ring-3). */
8317 pVCpu->hm.s.Event.fPending = false;
8318
8319 return VINF_SUCCESS;
8320}
8321
8322
8323/**
8324 * Prepares to run guest code in VT-x and we've committed to doing so. This
8325 * means there is no backing out to ring-3 or anywhere else at this
8326 * point.
8327 *
8328 * @param pVM Pointer to the VM.
8329 * @param pVCpu Pointer to the VMCPU.
8330 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8331 * out-of-sync. Make sure to update the required fields
8332 * before using them.
8333 * @param pVmxTransient Pointer to the VMX transient structure.
8334 *
8335 * @remarks Called with preemption disabled.
8336 * @remarks No-long-jump zone!!!
8337 */
8338static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8339{
8340 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8341 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8342 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8343
8344 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8345 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC); /* Indicate the start of guest execution. */
8346
8347 /*
8348 * If we are injecting events to a real-on-v86 mode guest, we may have to update
8349 * RIP and some other registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8350 * Reload only the necessary state, the assertion will catch if other parts of the code
8351 * change.
8352 */
8353 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
8354 {
8355 hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8356 hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8357 }
8358
8359#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8360 if (!CPUMIsGuestFPUStateActive(pVCpu))
8361 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8362 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8363#endif
8364
8365 if ( pVCpu->hm.s.fUseGuestFpu
8366 && !CPUMIsGuestFPUStateActive(pVCpu))
8367 {
8368 CPUMR0LoadGuestFPU(pVM, pVCpu, pMixedCtx);
8369 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
8370 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8371 }
8372
8373 /*
8374 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8375 */
8376 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8377 && pVCpu->hm.s.vmx.cMsrs > 0)
8378 {
8379 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8380 }
8381
8382 /*
8383 * Load the host state bits as we may've been preempted (only happens when
8384 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8385 */
8386 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8387 {
8388 /* This ASSUMES that pfnStartVM has been set up already. */
8389 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
8390 AssertRC(rc);
8391 STAM_COUNTER_INC(&pVCpu->hm.s.StatPreemptSaveHostState);
8392 }
8393 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
8394
8395 /*
8396 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
8397 */
8398 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
8399 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
8400 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8401
8402 /* Store status of the shared guest-host state at the time of VM-entry. */
8403#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS) && !defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
8404 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8405 {
8406 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8407 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8408 }
8409 else
8410#endif
8411 {
8412 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8413 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8414 }
8415 pVmxTransient->fWasGuestFPUStateActive = CPUMIsGuestFPUStateActive(pVCpu);
8416
8417 /*
8418 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8419 */
8420 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8421 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[0x80];
8422
8423 PHMGLOBALCPUINFO pCpu = HMR0GetCurrentCpu();
8424 RTCPUID idCurrentCpu = pCpu->idCpu;
8425 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8426 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8427 {
8428 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8429 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8430 }
8431
8432 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB-shootdowns, set this across the world switch. */
8433 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8434 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8435 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8436
8437 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8438
8439 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8440 to start executing. */
8441
8442 /*
8443 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8444 */
8445 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8446 {
8447 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8448 {
8449 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
8450 AssertRC(rc2);
8451 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
8452 bool fMsrUpdated = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMR0GetGuestTscAux(pVCpu),
8453 true /* fUpdateHostMsr */);
8454 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8455 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8457 }
8458 else
8459 {
8460 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8461 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8462 }
8463 }
8464
8465#ifdef VBOX_STRICT
8466 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8467 hmR0VmxCheckHostEferMsr(pVCpu);
8468 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8469#endif
8470#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8471 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
8472 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8473 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8474#endif
8475}
8476
8477
8478/**
8479 * Performs some essential restoration of state after running guest code in
8480 * VT-x.
8481 *
8482 * @param pVM Pointer to the VM.
8483 * @param pVCpu Pointer to the VMCPU.
8484 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8485 * out-of-sync. Make sure to update the required fields
8486 * before using them.
8487 * @param pVmxTransient Pointer to the VMX transient structure.
8488 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8489 *
8490 * @remarks Called with interrupts disabled, and returns with interrups enabled!
8491 *
8492 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8493 * unconditionally when it is safe to do so.
8494 */
8495static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8496{
8497 NOREF(pVM);
8498
8499 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8500
8501 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB-shootdowns. */
8502 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for TLB-shootdowns. */
8503 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
8504 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8505 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8506
8507 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8508 {
8509 /** @todo Find a way to fix hardcoding a guestimate. */
8510 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC()
8511 + pVCpu->hm.s.vmx.u64TSCOffset - 0x400 /* guestimate of world switch overhead in clock ticks */);
8512 }
8513
8514 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
8515 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8516 Assert(!(ASMGetFlags() & X86_EFL_IF));
8517 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8518
8519#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
8520 if (CPUMIsGuestFPUStateActive(pVCpu))
8521 {
8522 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8523 CPUMR0SaveGuestFPU(pVM, pVCpu, pMixedCtx);
8524 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
8525 }
8526#endif
8527
8528#if HC_ARCH_BITS == 64
8529 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8530#endif
8531 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8532#ifdef VBOX_STRICT
8533 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8534#endif
8535 ASMSetFlags(pVmxTransient->uEflags); /* Enable interrupts. */
8536 VMMRZCallRing3Enable(pVCpu); /* It is now safe to do longjmps to ring-3!!! */
8537
8538 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8539 uint32_t uExitReason;
8540 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8541 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8542 AssertRC(rc);
8543 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8544 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8545
8546 /* If the VMLAUNCH/VMRESUME failed, we can bail out early. This does -not- cover VMX_EXIT_ERR_*. */
8547 if (RT_UNLIKELY(rcVMRun != VINF_SUCCESS))
8548 {
8549 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
8550 pVmxTransient->fVMEntryFailed));
8551 return;
8552 }
8553
8554 if (RT_LIKELY(!pVmxTransient->fVMEntryFailed))
8555 {
8556 /* Update the guest interruptibility-state from the VMCS. */
8557 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
8558#if defined(HMVMX_SYNC_FULL_GUEST_STATE) || defined(HMVMX_SAVE_FULL_GUEST_STATE)
8559 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
8560 AssertRC(rc);
8561#endif
8562 /*
8563 * If the TPR was raised by the guest, it wouldn't cause a VM-exit immediately. Instead we sync the TPR lazily whenever
8564 * we eventually get a VM-exit for any reason. This maybe expensive as PDMApicSetTPR() can longjmp to ring-3 and which is
8565 * why it's done here as it's easier and no less efficient to deal with it here than making hmR0VmxSaveGuestState()
8566 * cope with longjmps safely (see VMCPU_FF_HM_UPDATE_CR3 handling).
8567 */
8568 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8569 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[0x80])
8570 {
8571 rc = PDMApicSetTPR(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[0x80]);
8572 AssertRC(rc);
8573 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
8574 }
8575 }
8576}
8577
8578
8579/**
8580 * Runs the guest code using VT-x the normal way.
8581 *
8582 * @returns VBox status code.
8583 * @param pVM Pointer to the VM.
8584 * @param pVCpu Pointer to the VMCPU.
8585 * @param pCtx Pointer to the guest-CPU context.
8586 *
8587 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8588 */
8589static int hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8590{
8591 VMXTRANSIENT VmxTransient;
8592 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8593 int rc = VERR_INTERNAL_ERROR_5;
8594 uint32_t cLoops = 0;
8595
8596 for (;; cLoops++)
8597 {
8598 Assert(!HMR0SuspendPending());
8599 HMVMX_ASSERT_CPU_SAFE();
8600
8601 /* Preparatory work for running guest code, this may force us to return
8602 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8603 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8604 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8605 if (rc != VINF_SUCCESS)
8606 break;
8607
8608 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8609 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8610 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8611
8612 /* Restore any residual host-state and save any bits shared between host
8613 and guest into the guest-CPU state. Re-enables interrupts! */
8614 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8615
8616 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8617 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8618 {
8619 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8620 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8621 return rc;
8622 }
8623
8624 /* Handle the VM-exit. */
8625 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8627 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8628 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8629 HMVMX_START_EXIT_DISPATCH_PROF();
8630#ifdef HMVMX_USE_FUNCTION_TABLE
8631 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8632#else
8633 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8634#endif
8635 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8636 if (rc != VINF_SUCCESS)
8637 break;
8638 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8639 {
8640 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8641 rc = VINF_EM_RAW_INTERRUPT;
8642 break;
8643 }
8644 }
8645
8646 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8647 return rc;
8648}
8649
8650
8651/**
8652 * Single steps guest code using VT-x.
8653 *
8654 * @returns VBox status code.
8655 * @param pVM Pointer to the VM.
8656 * @param pVCpu Pointer to the VMCPU.
8657 * @param pCtx Pointer to the guest-CPU context.
8658 *
8659 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
8660 */
8661static int hmR0VmxRunGuestCodeStep(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8662{
8663 VMXTRANSIENT VmxTransient;
8664 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8665 int rc = VERR_INTERNAL_ERROR_5;
8666 uint32_t cLoops = 0;
8667 uint16_t uCsStart = pCtx->cs.Sel;
8668 uint64_t uRipStart = pCtx->rip;
8669
8670 for (;; cLoops++)
8671 {
8672 Assert(!HMR0SuspendPending());
8673 HMVMX_ASSERT_CPU_SAFE();
8674
8675 /* Preparatory work for running guest code, this may force us to return
8676 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8677 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8678 rc = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient);
8679 if (rc != VINF_SUCCESS)
8680 break;
8681
8682 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
8683 rc = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
8684 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8685
8686 /* Restore any residual host-state and save any bits shared between host
8687 and guest into the guest-CPU state. Re-enables interrupts! */
8688 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rc);
8689
8690 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8691 if (RT_UNLIKELY(rc != VINF_SUCCESS))
8692 {
8693 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
8694 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rc, pCtx, &VmxTransient);
8695 return rc;
8696 }
8697
8698 /* Handle the VM-exit. */
8699 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8701 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8702 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
8703 HMVMX_START_EXIT_DISPATCH_PROF();
8704#ifdef HMVMX_USE_FUNCTION_TABLE
8705 rc = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8706#else
8707 rc = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8708#endif
8709 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
8710 if (rc != VINF_SUCCESS)
8711 break;
8712 else if (cLoops > pVM->hm.s.cMaxResumeLoops)
8713 {
8714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMaxResume);
8715 rc = VINF_EM_RAW_INTERRUPT;
8716 break;
8717 }
8718
8719 /*
8720 * Did the RIP change, if so, consider it a single step.
8721 * Otherwise, make sure one of the TFs gets set.
8722 */
8723 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
8724 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
8725 AssertRCReturn(rc2, rc2);
8726 if ( pCtx->rip != uRipStart
8727 || pCtx->cs.Sel != uCsStart)
8728 {
8729 rc = VINF_EM_DBG_STEPPED;
8730 break;
8731 }
8732 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
8733 }
8734
8735 /*
8736 * Clear the X86_EFL_TF if necessary.
8737 */
8738 if (pVCpu->hm.s.fClearTrapFlag)
8739 {
8740 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
8741 AssertRCReturn(rc2, rc2);
8742 pVCpu->hm.s.fClearTrapFlag = false;
8743 pCtx->eflags.Bits.u1TF = 0;
8744 }
8745 /** @todo there seems to be issues with the resume flag when the monitor trap
8746 * flag is pending without being used. Seen early in bios init when
8747 * accessing APIC page in protected mode. */
8748
8749 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8750 return rc;
8751}
8752
8753
8754/**
8755 * Runs the guest code using VT-x.
8756 *
8757 * @returns VBox status code.
8758 * @param pVM Pointer to the VM.
8759 * @param pVCpu Pointer to the VMCPU.
8760 * @param pCtx Pointer to the guest-CPU context.
8761 */
8762VMMR0DECL(int) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8763{
8764 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8765 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
8766 HMVMX_ASSERT_PREEMPT_SAFE();
8767
8768 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
8769
8770 int rc;
8771 if (!pVCpu->hm.s.fSingleInstruction && !DBGFIsStepping(pVCpu))
8772 rc = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
8773 else
8774 rc = hmR0VmxRunGuestCodeStep(pVM, pVCpu, pCtx);
8775
8776 if (rc == VERR_EM_INTERPRETER)
8777 rc = VINF_EM_RAW_EMULATE_INSTR;
8778 else if (rc == VINF_EM_RESET)
8779 rc = VINF_EM_TRIPLE_FAULT;
8780
8781 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rc);
8782 if (RT_FAILURE(rc2))
8783 {
8784 pVCpu->hm.s.u32HMError = rc;
8785 rc = rc2;
8786 }
8787 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
8788 return rc;
8789}
8790
8791
8792#ifndef HMVMX_USE_FUNCTION_TABLE
8793DECLINLINE(int) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
8794{
8795#ifdef DEBUG_ramshankar
8796# define SVVMCS() do { int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); } while (0)
8797# define LDVMCS() do { HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); } while (0)
8798#endif
8799 int rc;
8800 switch (rcReason)
8801 {
8802 case VMX_EXIT_EPT_MISCONFIG: /* SVVMCS(); */ rc = hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8803 case VMX_EXIT_EPT_VIOLATION: /* SVVMCS(); */ rc = hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8804 case VMX_EXIT_IO_INSTR: /* SVVMCS(); */ rc = hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8805 case VMX_EXIT_CPUID: /* SVVMCS(); */ rc = hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8806 case VMX_EXIT_RDTSC: /* SVVMCS(); */ rc = hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8807 case VMX_EXIT_RDTSCP: /* SVVMCS(); */ rc = hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8808 case VMX_EXIT_APIC_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8809 case VMX_EXIT_XCPT_OR_NMI: /* SVVMCS(); */ rc = hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8810 case VMX_EXIT_MOV_CRX: /* SVVMCS(); */ rc = hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8811 case VMX_EXIT_EXT_INT: /* SVVMCS(); */ rc = hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8812 case VMX_EXIT_INT_WINDOW: /* SVVMCS(); */ rc = hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8813 case VMX_EXIT_MWAIT: /* SVVMCS(); */ rc = hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8814 case VMX_EXIT_MONITOR: /* SVVMCS(); */ rc = hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8815 case VMX_EXIT_TASK_SWITCH: /* SVVMCS(); */ rc = hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8816 case VMX_EXIT_PREEMPT_TIMER: /* SVVMCS(); */ rc = hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8817 case VMX_EXIT_RDMSR: /* SVVMCS(); */ rc = hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8818 case VMX_EXIT_WRMSR: /* SVVMCS(); */ rc = hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8819 case VMX_EXIT_MOV_DRX: /* SVVMCS(); */ rc = hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8820 case VMX_EXIT_TPR_BELOW_THRESHOLD: /* SVVMCS(); */ rc = hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8821 case VMX_EXIT_HLT: /* SVVMCS(); */ rc = hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8822 case VMX_EXIT_INVD: /* SVVMCS(); */ rc = hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8823 case VMX_EXIT_INVLPG: /* SVVMCS(); */ rc = hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8824 case VMX_EXIT_RSM: /* SVVMCS(); */ rc = hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8825 case VMX_EXIT_MTF: /* SVVMCS(); */ rc = hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8826 case VMX_EXIT_PAUSE: /* SVVMCS(); */ rc = hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8827 case VMX_EXIT_XDTR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8828 case VMX_EXIT_TR_ACCESS: /* SVVMCS(); */ rc = hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8829 case VMX_EXIT_WBINVD: /* SVVMCS(); */ rc = hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8830 case VMX_EXIT_XSETBV: /* SVVMCS(); */ rc = hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8831 case VMX_EXIT_RDRAND: /* SVVMCS(); */ rc = hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8832 case VMX_EXIT_INVPCID: /* SVVMCS(); */ rc = hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8833 case VMX_EXIT_GETSEC: /* SVVMCS(); */ rc = hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8834 case VMX_EXIT_RDPMC: /* SVVMCS(); */ rc = hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient); /* LDVMCS(); */ break;
8835
8836 case VMX_EXIT_TRIPLE_FAULT: rc = hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient); break;
8837 case VMX_EXIT_NMI_WINDOW: rc = hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient); break;
8838 case VMX_EXIT_INIT_SIGNAL: rc = hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient); break;
8839 case VMX_EXIT_SIPI: rc = hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient); break;
8840 case VMX_EXIT_IO_SMI: rc = hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8841 case VMX_EXIT_SMI: rc = hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient); break;
8842 case VMX_EXIT_ERR_MSR_LOAD: rc = hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient); break;
8843 case VMX_EXIT_ERR_INVALID_GUEST_STATE: rc = hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient); break;
8844 case VMX_EXIT_ERR_MACHINE_CHECK: rc = hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient); break;
8845
8846 case VMX_EXIT_VMCALL:
8847 case VMX_EXIT_VMCLEAR:
8848 case VMX_EXIT_VMLAUNCH:
8849 case VMX_EXIT_VMPTRLD:
8850 case VMX_EXIT_VMPTRST:
8851 case VMX_EXIT_VMREAD:
8852 case VMX_EXIT_VMRESUME:
8853 case VMX_EXIT_VMWRITE:
8854 case VMX_EXIT_VMXOFF:
8855 case VMX_EXIT_VMXON:
8856 case VMX_EXIT_INVEPT:
8857 case VMX_EXIT_INVVPID:
8858 case VMX_EXIT_VMFUNC:
8859 rc = hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
8860 break;
8861 default:
8862 rc = hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
8863 break;
8864 }
8865 return rc;
8866}
8867#endif
8868
8869#ifdef DEBUG
8870/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
8871# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
8872 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
8873
8874# define HMVMX_ASSERT_PREEMPT_CPUID() \
8875 do \
8876 { \
8877 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
8878 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
8879 } while (0)
8880
8881# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8882 do { \
8883 AssertPtr(pVCpu); \
8884 AssertPtr(pMixedCtx); \
8885 AssertPtr(pVmxTransient); \
8886 Assert(pVmxTransient->fVMEntryFailed == false); \
8887 Assert(ASMIntAreEnabled()); \
8888 HMVMX_ASSERT_PREEMPT_SAFE(); \
8889 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
8890 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
8891 HMVMX_ASSERT_PREEMPT_SAFE(); \
8892 if (VMMR0IsLogFlushDisabled(pVCpu)) \
8893 HMVMX_ASSERT_PREEMPT_CPUID(); \
8894 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8895 } while (0)
8896
8897# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
8898 do { \
8899 Log4Func(("\n")); \
8900 } while (0)
8901#else /* Release builds */
8902# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
8903 do { \
8904 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
8905 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
8906 } while (0)
8907# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
8908#endif
8909
8910
8911/**
8912 * Advances the guest RIP after reading it from the VMCS.
8913 *
8914 * @returns VBox status code.
8915 * @param pVCpu Pointer to the VMCPU.
8916 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
8917 * out-of-sync. Make sure to update the required fields
8918 * before using them.
8919 * @param pVmxTransient Pointer to the VMX transient structure.
8920 *
8921 * @remarks No-long-jump zone!!!
8922 */
8923DECLINLINE(int) hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8924{
8925 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
8926 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
8927 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8928 AssertRCReturn(rc, rc);
8929
8930 pMixedCtx->rip += pVmxTransient->cbInstr;
8931 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
8932
8933 /*
8934 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
8935 * pending debug exception field as it takes care of priority of events.
8936 *
8937 * See Intel spec. 32.2.1 "Debug Exceptions".
8938 */
8939 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
8940
8941 return rc;
8942}
8943
8944
8945/**
8946 * Tries to determine what part of the guest-state VT-x has deemed as invalid
8947 * and update error record fields accordingly.
8948 *
8949 * @return VMX_IGS_* return codes.
8950 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
8951 * wrong with the guest state.
8952 *
8953 * @param pVM Pointer to the VM.
8954 * @param pVCpu Pointer to the VMCPU.
8955 * @param pCtx Pointer to the guest-CPU state.
8956 *
8957 * @remarks This function assumes our cache of the VMCS controls
8958 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
8959 */
8960static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8961{
8962#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
8963#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
8964 uError = (err); \
8965 break; \
8966 } else do { } while (0)
8967
8968 int rc;
8969 uint32_t uError = VMX_IGS_ERROR;
8970 uint32_t u32Val;
8971 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
8972
8973 do
8974 {
8975 /*
8976 * CR0.
8977 */
8978 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8979 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
8980 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
8981 See Intel spec. 26.3.1 "Checks on guest Guest Control Registers, Debug Registers and MSRs." */
8982 if (fUnrestrictedGuest)
8983 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
8984
8985 uint32_t u32GuestCR0;
8986 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
8987 AssertRCBreak(rc);
8988 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
8989 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
8990 if ( !fUnrestrictedGuest
8991 && (u32GuestCR0 & X86_CR0_PG)
8992 && !(u32GuestCR0 & X86_CR0_PE))
8993 {
8994 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
8995 }
8996
8997 /*
8998 * CR4.
8999 */
9000 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9001 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
9002
9003 uint32_t u32GuestCR4;
9004 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
9005 AssertRCBreak(rc);
9006 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
9007 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
9008
9009 /*
9010 * IA32_DEBUGCTL MSR.
9011 */
9012 uint64_t u64Val;
9013 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
9014 AssertRCBreak(rc);
9015 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9016 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
9017 {
9018 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
9019 }
9020 uint64_t u64DebugCtlMsr = u64Val;
9021
9022#ifdef VBOX_STRICT
9023 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
9024 AssertRCBreak(rc);
9025 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
9026#endif
9027 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
9028
9029 /*
9030 * RIP and RFLAGS.
9031 */
9032 uint32_t u32Eflags;
9033#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9034 if (HMVMX_IS_64BIT_HOST_MODE())
9035 {
9036 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
9037 AssertRCBreak(rc);
9038 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
9039 if ( !fLongModeGuest
9040 || !pCtx->cs.Attr.n.u1Long)
9041 {
9042 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
9043 }
9044 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
9045 * must be identical if the "IA32e mode guest" VM-entry control is 1
9046 * and CS.L is 1. No check applies if the CPU supports 64
9047 * linear-address bits. */
9048
9049 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
9050 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
9051 AssertRCBreak(rc);
9052 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
9053 VMX_IGS_RFLAGS_RESERVED);
9054 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9055 u32Eflags = u64Val;
9056 }
9057 else
9058#endif
9059 {
9060 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
9061 AssertRCBreak(rc);
9062 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
9063 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
9064 }
9065
9066 if ( fLongModeGuest
9067 || ( fUnrestrictedGuest
9068 && !(u32GuestCR0 & X86_CR0_PE)))
9069 {
9070 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
9071 }
9072
9073 uint32_t u32EntryInfo;
9074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
9075 AssertRCBreak(rc);
9076 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9077 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9078 {
9079 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
9080 }
9081
9082 /*
9083 * 64-bit checks.
9084 */
9085#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9086 if (HMVMX_IS_64BIT_HOST_MODE())
9087 {
9088 if ( fLongModeGuest
9089 && !fUnrestrictedGuest)
9090 {
9091 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
9092 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
9093 }
9094
9095 if ( !fLongModeGuest
9096 && (u32GuestCR4 & X86_CR4_PCIDE))
9097 {
9098 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
9099 }
9100
9101 /** @todo CR3 field must be such that bits 63:52 and bits in the range
9102 * 51:32 beyond the processor's physical-address width are 0. */
9103
9104 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
9105 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
9106 {
9107 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
9108 }
9109
9110 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
9111 AssertRCBreak(rc);
9112 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
9113
9114 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
9115 AssertRCBreak(rc);
9116 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
9117 }
9118#endif
9119
9120 /*
9121 * PERF_GLOBAL MSR.
9122 */
9123 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
9124 {
9125 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
9126 AssertRCBreak(rc);
9127 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
9128 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
9129 }
9130
9131 /*
9132 * PAT MSR.
9133 */
9134 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
9135 {
9136 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
9137 AssertRCBreak(rc);
9138 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
9139 for (unsigned i = 0; i < 8; i++)
9140 {
9141 uint8_t u8Val = (u64Val & 0x7);
9142 if ( u8Val != 0 /* UC */
9143 || u8Val != 1 /* WC */
9144 || u8Val != 4 /* WT */
9145 || u8Val != 5 /* WP */
9146 || u8Val != 6 /* WB */
9147 || u8Val != 7 /* UC- */)
9148 {
9149 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
9150 }
9151 u64Val >>= 3;
9152 }
9153 }
9154
9155 /*
9156 * EFER MSR.
9157 */
9158 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
9159 {
9160 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
9161 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
9162 AssertRCBreak(rc);
9163 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
9164 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
9165 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
9166 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
9167 HMVMX_CHECK_BREAK( fUnrestrictedGuest
9168 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u32GuestCR0 & X86_CR0_PG),
9169 VMX_IGS_EFER_LMA_PG_MISMATCH);
9170 }
9171
9172 /*
9173 * Segment registers.
9174 */
9175 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9176 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
9177 if (!(u32Eflags & X86_EFL_VM))
9178 {
9179 /* CS */
9180 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
9181 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
9182 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
9183 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
9184 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9185 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
9186 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
9187 /* CS cannot be loaded with NULL in protected mode. */
9188 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
9189 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
9190 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
9191 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
9192 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
9193 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
9194 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
9195 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
9196 else
9197 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
9198
9199 /* SS */
9200 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9201 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
9202 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
9203 if ( !(pCtx->cr0 & X86_CR0_PE)
9204 || pCtx->cs.Attr.n.u4Type == 3)
9205 {
9206 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
9207 }
9208 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
9209 {
9210 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
9211 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
9212 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
9213 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
9214 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
9215 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9216 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
9217 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
9218 }
9219
9220 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
9221 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
9222 {
9223 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
9224 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
9225 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9226 || pCtx->ds.Attr.n.u4Type > 11
9227 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9228 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
9229 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
9230 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
9231 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9232 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
9233 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
9234 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9235 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
9236 }
9237 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
9238 {
9239 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
9240 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
9241 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9242 || pCtx->es.Attr.n.u4Type > 11
9243 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
9244 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
9245 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
9246 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
9247 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9248 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
9249 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
9250 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9251 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
9252 }
9253 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
9254 {
9255 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
9256 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
9257 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9258 || pCtx->fs.Attr.n.u4Type > 11
9259 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
9260 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
9261 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
9262 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
9263 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9264 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
9265 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
9266 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9267 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
9268 }
9269 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
9270 {
9271 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
9272 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
9273 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
9274 || pCtx->gs.Attr.n.u4Type > 11
9275 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
9276 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
9277 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
9278 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
9279 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9280 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
9281 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
9282 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
9283 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
9284 }
9285 /* 64-bit capable CPUs. */
9286#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9287 if (HMVMX_IS_64BIT_HOST_MODE())
9288 {
9289 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9290 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9291 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9292 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9293 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9294 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9295 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9296 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9297 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9298 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9299 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9300 }
9301#endif
9302 }
9303 else
9304 {
9305 /* V86 mode checks. */
9306 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
9307 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9308 {
9309 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
9310 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
9311 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
9312 }
9313 else
9314 {
9315 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
9316 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
9317 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
9318 }
9319
9320 /* CS */
9321 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
9322 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
9323 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
9324 /* SS */
9325 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
9326 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
9327 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
9328 /* DS */
9329 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
9330 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
9331 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
9332 /* ES */
9333 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
9334 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
9335 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
9336 /* FS */
9337 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
9338 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
9339 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
9340 /* GS */
9341 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
9342 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
9343 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
9344 /* 64-bit capable CPUs. */
9345#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9346 if (HMVMX_IS_64BIT_HOST_MODE())
9347 {
9348 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
9349 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
9350 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
9351 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
9352 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
9353 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
9354 VMX_IGS_LONGMODE_SS_BASE_INVALID);
9355 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
9356 VMX_IGS_LONGMODE_DS_BASE_INVALID);
9357 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
9358 VMX_IGS_LONGMODE_ES_BASE_INVALID);
9359 }
9360#endif
9361 }
9362
9363 /*
9364 * TR.
9365 */
9366 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
9367 /* 64-bit capable CPUs. */
9368#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9369 if (HMVMX_IS_64BIT_HOST_MODE())
9370 {
9371 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
9372 }
9373#endif
9374 if (fLongModeGuest)
9375 {
9376 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
9377 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
9378 }
9379 else
9380 {
9381 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
9382 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
9383 VMX_IGS_TR_ATTR_TYPE_INVALID);
9384 }
9385 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
9386 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
9387 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
9388 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
9389 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9390 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
9391 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
9392 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
9393
9394 /*
9395 * GDTR and IDTR.
9396 */
9397#if HC_ARCH_BITS == 64 || defined(VBOX_WITH_HYBRID_32BIT_KERNEL)
9398 if (HMVMX_IS_64BIT_HOST_MODE())
9399 {
9400 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
9401 AssertRCBreak(rc);
9402 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
9403
9404 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
9405 AssertRCBreak(rc);
9406 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
9407 }
9408#endif
9409
9410 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
9411 AssertRCBreak(rc);
9412 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9413
9414 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
9415 AssertRCBreak(rc);
9416 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
9417
9418 /*
9419 * Guest Non-Register State.
9420 */
9421 /* Activity State. */
9422 uint32_t u32ActivityState;
9423 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
9424 AssertRCBreak(rc);
9425 HMVMX_CHECK_BREAK( !u32ActivityState
9426 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
9427 VMX_IGS_ACTIVITY_STATE_INVALID);
9428 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
9429 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
9430 uint32_t u32IntrState;
9431 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
9432 AssertRCBreak(rc);
9433 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
9434 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9435 {
9436 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
9437 }
9438
9439 /** @todo Activity state and injecting interrupts. Left as a todo since we
9440 * currently don't use activity states but ACTIVE. */
9441
9442 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9443 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
9444
9445 /* Guest interruptibility-state. */
9446 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
9447 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9448 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
9449 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
9450 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9451 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
9452 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
9453 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9454 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
9455 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
9456 {
9457 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
9458 {
9459 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9460 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9461 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
9462 }
9463 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9464 {
9465 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
9466 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
9467 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
9468 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
9469 }
9470 }
9471 /** @todo Assumes the processor is not in SMM. */
9472 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9473 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
9474 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
9475 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
9476 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
9477 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
9478 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
9479 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9480 {
9481 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
9482 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
9483 }
9484
9485 /* Pending debug exceptions. */
9486 if (HMVMX_IS_64BIT_HOST_MODE())
9487 {
9488 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
9489 AssertRCBreak(rc);
9490 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
9491 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
9492 u32Val = u64Val; /* For pending debug exceptions checks below. */
9493 }
9494 else
9495 {
9496 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
9497 AssertRCBreak(rc);
9498 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
9499 HMVMX_CHECK_BREAK(!(u64Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
9500 }
9501
9502 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
9503 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
9504 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
9505 {
9506 if ( (u32Eflags & X86_EFL_TF)
9507 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9508 {
9509 /* Bit 14 is PendingDebug.BS. */
9510 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
9511 }
9512 if ( !(u32Eflags & X86_EFL_TF)
9513 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
9514 {
9515 /* Bit 14 is PendingDebug.BS. */
9516 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
9517 }
9518 }
9519
9520 /* VMCS link pointer. */
9521 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
9522 AssertRCBreak(rc);
9523 if (u64Val != UINT64_C(0xffffffffffffffff))
9524 {
9525 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
9526 /** @todo Bits beyond the processor's physical-address width MBZ. */
9527 /** @todo 32-bit located in memory referenced by value of this field (as a
9528 * physical address) must contain the processor's VMCS revision ID. */
9529 /** @todo SMM checks. */
9530 }
9531
9532 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
9533 * not using Nested Paging? */
9534 if ( pVM->hm.s.fNestedPaging
9535 && !fLongModeGuest
9536 && CPUMIsGuestInPAEModeEx(pCtx))
9537 {
9538 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
9539 AssertRCBreak(rc);
9540 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9541
9542 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
9543 AssertRCBreak(rc);
9544 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9545
9546 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
9547 AssertRCBreak(rc);
9548 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9549
9550 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
9551 AssertRCBreak(rc);
9552 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
9553 }
9554
9555 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
9556 if (uError == VMX_IGS_ERROR)
9557 uError = VMX_IGS_REASON_NOT_FOUND;
9558 } while (0);
9559
9560 pVCpu->hm.s.u32HMError = uError;
9561 return uError;
9562
9563#undef HMVMX_ERROR_BREAK
9564#undef HMVMX_CHECK_BREAK
9565}
9566
9567/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9568/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
9569/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
9570
9571/** @name VM-exit handlers.
9572 * @{
9573 */
9574
9575/**
9576 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
9577 */
9578HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9579{
9580 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9581 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
9582 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
9583 if (VMMR0ThreadCtxHooksAreRegistered(pVCpu))
9584 return VINF_SUCCESS;
9585 return VINF_EM_RAW_INTERRUPT;
9586}
9587
9588
9589/**
9590 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
9591 */
9592HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9593{
9594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9595 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
9596
9597 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9598 AssertRCReturn(rc, rc);
9599
9600 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9601 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
9602 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
9603 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
9604
9605 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9606 {
9607 /*
9608 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
9609 * anything we inject is not going to cause a VM-exit directly for the event being injected.
9610 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
9611 *
9612 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
9613 */
9614 VMXDispatchHostNmi();
9615 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
9616 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9617 return VINF_SUCCESS;
9618 }
9619
9620 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
9621 rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
9622 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
9623 {
9624 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9625 return VINF_SUCCESS;
9626 }
9627 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
9628 {
9629 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9630 return rc;
9631 }
9632
9633 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
9634 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
9635 switch (uIntType)
9636 {
9637 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
9638 Assert(uVector == X86_XCPT_DB);
9639 /* no break */
9640 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
9641 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
9642 /* no break */
9643 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9644 {
9645 switch (uVector)
9646 {
9647 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
9648 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
9649 case X86_XCPT_NM: rc = hmR0VmxExitXcptNM(pVCpu, pMixedCtx, pVmxTransient); break;
9650 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
9651 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
9652 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
9653#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
9654 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
9655 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9656 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
9657 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9658 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
9659 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9660 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
9661 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9662 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
9663 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9664 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
9665 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
9666#endif
9667 default:
9668 {
9669 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9670 AssertRCReturn(rc, rc);
9671
9672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
9673 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
9674 {
9675 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
9676 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
9677 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
9678
9679 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
9680 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9681 AssertRCReturn(rc, rc);
9682 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
9683 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
9684 0 /* GCPtrFaultAddress */);
9685 AssertRCReturn(rc, rc);
9686 }
9687 else
9688 {
9689 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
9690 pVCpu->hm.s.u32HMError = uVector;
9691 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
9692 }
9693 break;
9694 }
9695 }
9696 break;
9697 }
9698
9699 default:
9700 {
9701 pVCpu->hm.s.u32HMError = uExitIntInfo;
9702 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
9703 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
9704 break;
9705 }
9706 }
9707 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
9708 return rc;
9709}
9710
9711
9712/**
9713 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
9714 */
9715HMVMX_EXIT_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9716{
9717 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9718
9719 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
9720 hmR0VmxClearIntWindowExitVmcs(pVCpu);
9721
9722 /* Deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectEvent() and resume guest execution. */
9723 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
9724 return VINF_SUCCESS;
9725}
9726
9727
9728/**
9729 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
9730 */
9731HMVMX_EXIT_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9732{
9733 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9734 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
9735 HMVMX_RETURN_UNEXPECTED_EXIT();
9736}
9737
9738
9739/**
9740 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
9741 */
9742HMVMX_EXIT_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9743{
9744 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9745 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
9746 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9747}
9748
9749
9750/**
9751 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
9752 */
9753HMVMX_EXIT_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9754{
9755 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9756 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
9757 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9758}
9759
9760
9761/**
9762 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
9763 */
9764HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9765{
9766 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9767 PVM pVM = pVCpu->CTX_SUFF(pVM);
9768 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9769 if (RT_LIKELY(rc == VINF_SUCCESS))
9770 {
9771 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9772 Assert(pVmxTransient->cbInstr == 2);
9773 }
9774 else
9775 {
9776 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
9777 rc = VERR_EM_INTERPRETER;
9778 }
9779 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
9780 return rc;
9781}
9782
9783
9784/**
9785 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
9786 */
9787HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9788{
9789 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9790 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
9791 AssertRCReturn(rc, rc);
9792
9793 if (pMixedCtx->cr4 & X86_CR4_SMXE)
9794 return VINF_EM_RAW_EMULATE_INSTR;
9795
9796 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
9797 HMVMX_RETURN_UNEXPECTED_EXIT();
9798}
9799
9800
9801/**
9802 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
9803 */
9804HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9805{
9806 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9807 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9808 AssertRCReturn(rc, rc);
9809
9810 PVM pVM = pVCpu->CTX_SUFF(pVM);
9811 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9812 if (RT_LIKELY(rc == VINF_SUCCESS))
9813 {
9814 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9815 Assert(pVmxTransient->cbInstr == 2);
9816 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9817 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9818 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9819 }
9820 else
9821 {
9822 AssertMsgFailed(("hmR0VmxExitRdtsc: EMInterpretRdtsc failed with %Rrc\n", rc));
9823 rc = VERR_EM_INTERPRETER;
9824 }
9825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9826 return rc;
9827}
9828
9829
9830/**
9831 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
9832 */
9833HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9834{
9835 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9836 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9837 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
9838 AssertRCReturn(rc, rc);
9839
9840 PVM pVM = pVCpu->CTX_SUFF(pVM);
9841 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
9842 if (RT_LIKELY(rc == VINF_SUCCESS))
9843 {
9844 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9845 Assert(pVmxTransient->cbInstr == 3);
9846 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
9847 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
9848 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9849 }
9850 else
9851 {
9852 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
9853 rc = VERR_EM_INTERPRETER;
9854 }
9855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
9856 return rc;
9857}
9858
9859
9860/**
9861 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
9862 */
9863HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9864{
9865 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9866 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /** @todo review if CR4 is really required by EM. */
9867 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx); /** @todo review if CR0 is really required by EM. */
9868 AssertRCReturn(rc, rc);
9869
9870 PVM pVM = pVCpu->CTX_SUFF(pVM);
9871 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9872 if (RT_LIKELY(rc == VINF_SUCCESS))
9873 {
9874 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9875 Assert(pVmxTransient->cbInstr == 2);
9876 }
9877 else
9878 {
9879 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
9880 rc = VERR_EM_INTERPRETER;
9881 }
9882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
9883 return rc;
9884}
9885
9886
9887/**
9888 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
9889 */
9890HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9891{
9892 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9893 PVM pVM = pVCpu->CTX_SUFF(pVM);
9894 Assert(!pVM->hm.s.fNestedPaging);
9895
9896 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9897 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
9898 AssertRCReturn(rc, rc);
9899
9900 VBOXSTRICTRC rc2 = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
9901 rc = VBOXSTRICTRC_VAL(rc2);
9902 if (RT_LIKELY(rc == VINF_SUCCESS))
9903 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9904 else
9905 {
9906 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
9907 pVmxTransient->uExitQualification, rc));
9908 }
9909 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
9910 return rc;
9911}
9912
9913
9914/**
9915 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
9916 */
9917HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9918{
9919 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9920 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9921 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9922 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9923 AssertRCReturn(rc, rc);
9924
9925 PVM pVM = pVCpu->CTX_SUFF(pVM);
9926 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9927 if (RT_LIKELY(rc == VINF_SUCCESS))
9928 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9929 else
9930 {
9931 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
9932 rc = VERR_EM_INTERPRETER;
9933 }
9934 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
9935 return rc;
9936}
9937
9938
9939/**
9940 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
9941 */
9942HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9943{
9944 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9945 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9946 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9947 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
9948 AssertRCReturn(rc, rc);
9949
9950 PVM pVM = pVCpu->CTX_SUFF(pVM);
9951 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
9952 rc = VBOXSTRICTRC_VAL(rc2);
9953 if (RT_LIKELY( rc == VINF_SUCCESS
9954 || rc == VINF_EM_HALT))
9955 {
9956 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
9957 AssertRCReturn(rc3, rc3);
9958
9959 if ( rc == VINF_EM_HALT
9960 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
9961 {
9962 rc = VINF_SUCCESS;
9963 }
9964 }
9965 else
9966 {
9967 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
9968 rc = VERR_EM_INTERPRETER;
9969 }
9970 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
9971 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
9972 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
9973 return rc;
9974}
9975
9976
9977/**
9978 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
9979 */
9980HMVMX_EXIT_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9981{
9982 /*
9983 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
9984 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
9985 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
9986 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
9987 */
9988 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
9989 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
9990 HMVMX_RETURN_UNEXPECTED_EXIT();
9991}
9992
9993
9994/**
9995 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
9996 */
9997HMVMX_EXIT_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9998{
9999 /*
10000 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
10001 * root operation. Only an STM (SMM transfer monitor) would get this exit when we (the executive monitor) execute a VMCALL
10002 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
10003 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
10004 */
10005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10006 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10007 HMVMX_RETURN_UNEXPECTED_EXIT();
10008}
10009
10010
10011/**
10012 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
10013 */
10014HMVMX_EXIT_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10015{
10016 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
10017 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10018 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10019 HMVMX_RETURN_UNEXPECTED_EXIT();
10020}
10021
10022
10023/**
10024 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
10025 */
10026HMVMX_EXIT_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10027{
10028 /*
10029 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
10030 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
10031 * See Intel spec. 25.3 "Other Causes of VM-exits".
10032 */
10033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10034 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10035 HMVMX_RETURN_UNEXPECTED_EXIT();
10036}
10037
10038
10039/**
10040 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
10041 * VM-exit.
10042 */
10043HMVMX_EXIT_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10044{
10045 /*
10046 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
10047 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
10048 *
10049 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
10050 * See Intel spec. "23.8 Restrictions on VMX operation".
10051 */
10052 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10053 return VINF_SUCCESS;
10054}
10055
10056
10057/**
10058 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
10059 * VM-exit.
10060 */
10061HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10062{
10063 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10064 return VINF_EM_RESET;
10065}
10066
10067
10068/**
10069 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
10070 */
10071HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10072{
10073 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10074 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
10075 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10076 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10077 AssertRCReturn(rc, rc);
10078
10079 pMixedCtx->rip++;
10080 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10081 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
10082 rc = VINF_SUCCESS;
10083 else
10084 rc = VINF_EM_HALT;
10085
10086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
10087 return rc;
10088}
10089
10090
10091/**
10092 * VM-exit handler for instructions that result in a #UD exception delivered to
10093 * the guest.
10094 */
10095HMVMX_EXIT_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10096{
10097 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10098 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
10099 return VINF_SUCCESS;
10100}
10101
10102
10103/**
10104 * VM-exit handler for expiry of the VMX preemption timer.
10105 */
10106HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10107{
10108 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10109
10110 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
10111 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10112
10113 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
10114 PVM pVM = pVCpu->CTX_SUFF(pVM);
10115 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
10116 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
10117 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
10118}
10119
10120
10121/**
10122 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
10123 */
10124HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10125{
10126 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10127
10128 /* We expose XSETBV to the guest, fallback to the recompiler for emulation. */
10129 /** @todo check if XSETBV is supported by the recompiler. */
10130 return VERR_EM_INTERPRETER;
10131}
10132
10133
10134/**
10135 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
10136 */
10137HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10138{
10139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10140
10141 /* The guest should not invalidate the host CPU's TLBs, fallback to recompiler. */
10142 /** @todo implement EMInterpretInvpcid() */
10143 return VERR_EM_INTERPRETER;
10144}
10145
10146
10147/**
10148 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
10149 * Error VM-exit.
10150 */
10151HMVMX_EXIT_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10152{
10153 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10154 AssertRCReturn(rc, rc);
10155
10156 rc = hmR0VmxCheckVmcsCtls(pVCpu);
10157 AssertRCReturn(rc, rc);
10158
10159 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10160 NOREF(uInvalidReason);
10161
10162#ifdef VBOX_STRICT
10163 uint32_t uIntrState;
10164 HMVMXHCUINTREG uHCReg;
10165 uint64_t u64Val;
10166 uint32_t u32Val;
10167
10168 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
10169 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
10170 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
10171 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
10172 AssertRCReturn(rc, rc);
10173
10174 Log4(("uInvalidReason %u\n", uInvalidReason));
10175 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
10176 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
10177 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
10178 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
10179
10180 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
10181 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
10182 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
10183 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
10184 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
10185 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10186 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
10187 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
10188 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
10189 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
10190 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
10191 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
10192#else
10193 NOREF(pVmxTransient);
10194#endif
10195
10196 HMDumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
10197 return VERR_VMX_INVALID_GUEST_STATE;
10198}
10199
10200
10201/**
10202 * VM-exit handler for VM-entry failure due to an MSR-load
10203 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
10204 */
10205HMVMX_EXIT_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10206{
10207 NOREF(pVmxTransient);
10208 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10209 HMVMX_RETURN_UNEXPECTED_EXIT();
10210}
10211
10212
10213/**
10214 * VM-exit handler for VM-entry failure due to a machine-check event
10215 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
10216 */
10217HMVMX_EXIT_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10218{
10219 NOREF(pVmxTransient);
10220 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
10221 HMVMX_RETURN_UNEXPECTED_EXIT();
10222}
10223
10224
10225/**
10226 * VM-exit handler for all undefined reasons. Should never ever happen.. in
10227 * theory.
10228 */
10229HMVMX_EXIT_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10230{
10231 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
10232 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
10233 return VERR_VMX_UNDEFINED_EXIT_CODE;
10234}
10235
10236
10237/**
10238 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
10239 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
10240 * Conditional VM-exit.
10241 */
10242HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10243{
10244 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10245
10246 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
10247 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
10248 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
10249 return VERR_EM_INTERPRETER;
10250 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10251 HMVMX_RETURN_UNEXPECTED_EXIT();
10252}
10253
10254
10255/**
10256 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
10257 */
10258HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10259{
10260 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10261
10262 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
10263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
10264 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
10265 return VERR_EM_INTERPRETER;
10266 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10267 HMVMX_RETURN_UNEXPECTED_EXIT();
10268}
10269
10270
10271/**
10272 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
10273 */
10274HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10275{
10276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10277
10278 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
10279 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10280 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10281 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10282 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10283 {
10284 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10285 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10286 }
10287 AssertRCReturn(rc, rc);
10288 Log4(("CS:RIP=%04x:%#RX64 ECX=%X\n", pMixedCtx->cs.Sel, pMixedCtx->rip, pMixedCtx->ecx));
10289
10290#ifdef VBOX_STRICT
10291 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
10292 {
10293 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10294 {
10295 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10296 HMVMX_RETURN_UNEXPECTED_EXIT();
10297 }
10298# if HC_ARCH_BITS == 64
10299 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests
10300 && hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10301 {
10302 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10303 HMVMX_RETURN_UNEXPECTED_EXIT();
10304 }
10305# endif
10306 }
10307#endif
10308
10309 PVM pVM = pVCpu->CTX_SUFF(pVM);
10310 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10311 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
10312 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
10313 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
10314
10315 if (RT_LIKELY(rc == VINF_SUCCESS))
10316 {
10317 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10318 Assert(pVmxTransient->cbInstr == 2);
10319 }
10320 return rc;
10321}
10322
10323
10324/**
10325 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
10326 */
10327HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10328{
10329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10330 PVM pVM = pVCpu->CTX_SUFF(pVM);
10331 int rc = VINF_SUCCESS;
10332
10333 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
10334 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10335 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10336 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10337 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10338 {
10339 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
10340 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
10341 }
10342 AssertRCReturn(rc, rc);
10343 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
10344
10345 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
10346 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
10347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
10348
10349 if (RT_LIKELY(rc == VINF_SUCCESS))
10350 {
10351 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10352
10353 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
10354 if ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
10355 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END)
10356 {
10357 /* We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
10358 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
10359 EMInterpretWrmsr() changes it. */
10360 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10361 }
10362 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
10363 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
10364 else if (pMixedCtx->ecx == MSR_K6_EFER)
10365 {
10366 /*
10367 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
10368 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
10369 * the other bits as well, SCE and NXE. See @bugref{7368}.
10370 */
10371 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
10372 }
10373
10374 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
10375 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
10376 {
10377 switch (pMixedCtx->ecx)
10378 {
10379 case MSR_IA32_SYSENTER_CS: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
10380 case MSR_IA32_SYSENTER_EIP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
10381 case MSR_IA32_SYSENTER_ESP: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
10382 case MSR_K8_FS_BASE: /* no break */
10383 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
10384 default:
10385 {
10386 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10387 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
10388#if HC_ARCH_BITS == 64
10389 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10390 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_LAZY_MSRS);
10391#endif
10392 break;
10393 }
10394 }
10395 }
10396#ifdef VBOX_STRICT
10397 else
10398 {
10399 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
10400 switch (pMixedCtx->ecx)
10401 {
10402 case MSR_IA32_SYSENTER_CS:
10403 case MSR_IA32_SYSENTER_EIP:
10404 case MSR_IA32_SYSENTER_ESP:
10405 case MSR_K8_FS_BASE:
10406 case MSR_K8_GS_BASE:
10407 {
10408 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
10409 HMVMX_RETURN_UNEXPECTED_EXIT();
10410 }
10411
10412 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
10413 default:
10414 {
10415 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
10416 {
10417 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
10418 pMixedCtx->ecx));
10419 HMVMX_RETURN_UNEXPECTED_EXIT();
10420 }
10421
10422#if HC_ARCH_BITS == 64
10423 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
10424 {
10425 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
10426 HMVMX_RETURN_UNEXPECTED_EXIT();
10427 }
10428#endif
10429 break;
10430 }
10431 }
10432 }
10433#endif /* VBOX_STRICT */
10434 }
10435 return rc;
10436}
10437
10438
10439/**
10440 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
10441 */
10442HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10443{
10444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10445
10446 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT. */
10447 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
10448 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT)
10449 return VERR_EM_INTERPRETER;
10450 AssertMsgFailed(("Unexpected PAUSE exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10451 HMVMX_RETURN_UNEXPECTED_EXIT();
10452}
10453
10454
10455/**
10456 * VM-exit handler for when the TPR value is lowered below the specified
10457 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
10458 */
10459HMVMX_EXIT_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10460{
10461 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10462 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
10463
10464 /*
10465 * The TPR has already been updated, see hmR0VMXPostRunGuest(). RIP is also updated as part of the VM-exit by VT-x. Update
10466 * the threshold in the VMCS, deliver the pending interrupt via hmR0VmxPreRunGuest()->hmR0VmxInjectPendingEvent() and
10467 * resume guest execution.
10468 */
10469 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10470 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
10471 return VINF_SUCCESS;
10472}
10473
10474
10475/**
10476 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
10477 * VM-exit.
10478 *
10479 * @retval VINF_SUCCESS when guest execution can continue.
10480 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
10481 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
10482 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
10483 * recompiler.
10484 */
10485HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10486{
10487 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10488 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
10489 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10490 AssertRCReturn(rc, rc);
10491
10492 const RTGCUINTPTR uExitQualification = pVmxTransient->uExitQualification;
10493 const uint32_t uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
10494 PVM pVM = pVCpu->CTX_SUFF(pVM);
10495 switch (uAccessType)
10496 {
10497 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
10498 {
10499#if 0
10500 /* EMInterpretCRxWrite() references a lot of guest state (EFER, RFLAGS, Segment Registers, etc.) Sync entire state */
10501 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10502#else
10503 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10504 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10505 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10506#endif
10507 AssertRCReturn(rc, rc);
10508
10509 rc = EMInterpretCRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10510 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
10511 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
10512 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_PGM_SYNC_CR3);
10513
10514 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
10515 {
10516 case 0: /* CR0 */
10517 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10518 Log4(("CRX CR0 write rc=%d CR0=%#RX64\n", rc, pMixedCtx->cr0));
10519 break;
10520 case 2: /* CR2 */
10521 /* Nothing to do here, CR2 it's not part of the VMCS. */
10522 break;
10523 case 3: /* CR3 */
10524 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx));
10525 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
10526 Log4(("CRX CR3 write rc=%d CR3=%#RX64\n", rc, pMixedCtx->cr3));
10527 break;
10528 case 4: /* CR4 */
10529 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
10530 Log4(("CRX CR4 write rc=%d CR4=%#RX64\n", rc, pMixedCtx->cr4));
10531 break;
10532 case 8: /* CR8 */
10533 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10534 /* CR8 contains the APIC TPR. Was updated by EMInterpretCRxWrite(). */
10535 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_APIC_STATE);
10536 break;
10537 default:
10538 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
10539 break;
10540 }
10541
10542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10543 break;
10544 }
10545
10546 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
10547 {
10548 /* EMInterpretCRxRead() requires EFER MSR, CS. */
10549 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10550 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10551 AssertRCReturn(rc, rc);
10552 Assert( !pVM->hm.s.fNestedPaging
10553 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
10554 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
10555
10556 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
10557 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
10558 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
10559
10560 rc = EMInterpretCRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
10561 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
10562 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
10563 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
10564 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
10565 Log4(("CRX CR%d Read access rc=%d\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification), rc));
10566 break;
10567 }
10568
10569 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
10570 {
10571 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10572 AssertRCReturn(rc, rc);
10573 rc = EMInterpretCLTS(pVM, pVCpu);
10574 AssertRCReturn(rc, rc);
10575 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10576 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
10577 Log4(("CRX CLTS write rc=%d\n", rc));
10578 break;
10579 }
10580
10581 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
10582 {
10583 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
10584 AssertRCReturn(rc, rc);
10585 rc = EMInterpretLMSW(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
10586 if (RT_LIKELY(rc == VINF_SUCCESS))
10587 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
10588 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
10589 Log4(("CRX LMSW write rc=%d\n", rc));
10590 break;
10591 }
10592
10593 default:
10594 {
10595 AssertMsgFailed(("Invalid access-type in Mov CRx exit qualification %#x\n", uAccessType));
10596 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10597 }
10598 }
10599
10600 /* Validate possible error codes. */
10601 Assert(rc == VINF_SUCCESS || rc == VINF_PGM_CHANGE_MODE || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_SYNC_CR3
10602 || rc == VERR_VMX_UNEXPECTED_EXCEPTION);
10603 if (RT_SUCCESS(rc))
10604 {
10605 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
10606 AssertRCReturn(rc2, rc2);
10607 }
10608
10609 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
10610 return rc;
10611}
10612
10613
10614/**
10615 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
10616 * VM-exit.
10617 */
10618HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10619{
10620 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10621 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
10622
10623 int rc2 = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10624 rc2 |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10625 rc2 |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10626 rc2 |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
10627 rc2 |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
10628 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
10629 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
10630 AssertRCReturn(rc2, rc2);
10631
10632 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
10633 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
10634 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
10635 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
10636 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
10637 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
10638 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
10639 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_HMVMX_IPE_1);
10640
10641 /* I/O operation lookup arrays. */
10642 static const uint32_t s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
10643 static const uint32_t s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
10644
10645 VBOXSTRICTRC rcStrict;
10646 const uint32_t cbValue = s_aIOSizes[uIOWidth];
10647 const uint32_t cbInstr = pVmxTransient->cbInstr;
10648 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
10649 PVM pVM = pVCpu->CTX_SUFF(pVM);
10650 if (fIOString)
10651 {
10652#if 0 /* Not yet ready. IEM gurus with debian 32-bit guest without NP (on ATA reads). See @bugref{5752#c158} */
10653 /*
10654 * INS/OUTS - I/O String instruction.
10655 *
10656 * Use instruction-information if available, otherwise fall back on
10657 * interpreting the instruction.
10658 */
10659 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10660 AssertReturn(pMixedCtx->dx == uIOPort, VERR_HMVMX_IPE_2);
10661 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
10662 {
10663 rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10664 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10665 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10666 AssertRCReturn(rc2, rc2);
10667 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_HMVMX_IPE_3);
10668 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
10669 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
10670 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
10671 if (fIOWrite)
10672 {
10673 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
10674 pVmxTransient->ExitInstrInfo.StrIo.iSegReg);
10675 }
10676 else
10677 {
10678 /*
10679 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
10680 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
10681 * See Intel Instruction spec. for "INS".
10682 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
10683 */
10684 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr);
10685 }
10686 }
10687 else
10688 {
10689 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
10690 rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10691 AssertRCReturn(rc2, rc2);
10692 rcStrict = IEMExecOne(pVCpu);
10693 }
10694 /** @todo IEM needs to be setting these flags somehow. */
10695 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10696 fUpdateRipAlready = true;
10697#else
10698 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
10699 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
10700 if (RT_SUCCESS(rcStrict))
10701 {
10702 if (fIOWrite)
10703 {
10704 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10705 (DISCPUMODE)pDis->uAddrMode, cbValue);
10706 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
10707 }
10708 else
10709 {
10710 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
10711 (DISCPUMODE)pDis->uAddrMode, cbValue);
10712 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
10713 }
10714 }
10715 else
10716 {
10717 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP %#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->rip));
10718 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10719 }
10720#endif
10721 }
10722 else
10723 {
10724 /*
10725 * IN/OUT - I/O instruction.
10726 */
10727 Log4(("CS:RIP=%04x:%#RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
10728 const uint32_t uAndVal = s_aIOOpAnd[uIOWidth];
10729 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
10730 if (fIOWrite)
10731 {
10732 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
10733 if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10734 HMR0SavePendingIOPortWrite(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
10736 }
10737 else
10738 {
10739 uint32_t u32Result = 0;
10740 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
10741 if (IOM_SUCCESS(rcStrict))
10742 {
10743 /* Save result of I/O IN instr. in AL/AX/EAX. */
10744 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
10745 }
10746 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10747 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
10748 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
10749 }
10750 }
10751
10752 if (IOM_SUCCESS(rcStrict))
10753 {
10754 if (!fUpdateRipAlready)
10755 {
10756 pMixedCtx->rip += cbInstr;
10757 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10758 }
10759
10760 /*
10761 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
10762 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
10763 */
10764 if (fIOString)
10765 {
10766 /** @todo Single-step for INS/OUTS with REP prefix? */
10767 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
10768 }
10769 else if (fStepping)
10770 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
10771
10772 /*
10773 * If any I/O breakpoints are armed, we need to check if one triggered
10774 * and take appropriate action.
10775 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
10776 */
10777 rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
10778 AssertRCReturn(rc2, rc2);
10779
10780 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
10781 * execution engines about whether hyper BPs and such are pending. */
10782 uint32_t const uDr7 = pMixedCtx->dr[7];
10783 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
10784 && X86_DR7_ANY_RW_IO(uDr7)
10785 && (pMixedCtx->cr4 & X86_CR4_DE))
10786 || DBGFBpIsHwIoArmed(pVM)))
10787 {
10788 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
10789
10790 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
10791 VMMRZCallRing3Disable(pVCpu);
10792 HM_DISABLE_PREEMPT_IF_NEEDED();
10793
10794 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /*fDr6*/);
10795
10796 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
10797 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
10798 {
10799 /* Raise #DB. */
10800 if (fIsGuestDbgActive)
10801 ASMSetDR6(pMixedCtx->dr[6]);
10802 if (pMixedCtx->dr[7] != uDr7)
10803 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10804
10805 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
10806 }
10807 /* rcStrict is VINF_SUCCESS or in [VINF_EM_FIRST..VINF_EM_LAST]. */
10808 else if ( rcStrict2 != VINF_SUCCESS
10809 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
10810 rcStrict = rcStrict2;
10811
10812 HM_RESTORE_PREEMPT_IF_NEEDED();
10813 VMMRZCallRing3Enable(pVCpu);
10814 }
10815 }
10816
10817#ifdef DEBUG
10818 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
10819 Assert(!fIOWrite);
10820 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE)
10821 Assert(fIOWrite);
10822 else
10823 {
10824 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
10825 * statuses, that the VMM device and some others may return. See
10826 * IOM_SUCCESS() for guidance. */
10827 AssertMsg( RT_FAILURE(rcStrict)
10828 || rcStrict == VINF_SUCCESS
10829 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
10830 || rcStrict == VINF_EM_DBG_BREAKPOINT
10831 || rcStrict == VINF_EM_RAW_GUEST_TRAP
10832 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
10833 }
10834#endif
10835
10836 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
10837 return VBOXSTRICTRC_TODO(rcStrict);
10838}
10839
10840
10841/**
10842 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
10843 * VM-exit.
10844 */
10845HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10846{
10847 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10848
10849 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
10850 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10851 AssertRCReturn(rc, rc);
10852 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
10853 {
10854 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
10855 AssertRCReturn(rc, rc);
10856 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
10857 {
10858 uint32_t uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
10859
10860 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
10861 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
10862
10863 /* Save it as a pending event and it'll be converted to a TRPM event on the way out to ring-3. */
10864 Assert(!pVCpu->hm.s.Event.fPending);
10865 pVCpu->hm.s.Event.fPending = true;
10866 pVCpu->hm.s.Event.u64IntInfo = pVmxTransient->uIdtVectoringInfo;
10867 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
10868 AssertRCReturn(rc, rc);
10869 if (fErrorCodeValid)
10870 pVCpu->hm.s.Event.u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
10871 else
10872 pVCpu->hm.s.Event.u32ErrCode = 0;
10873 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
10874 && uVector == X86_XCPT_PF)
10875 {
10876 pVCpu->hm.s.Event.GCPtrFaultAddress = pMixedCtx->cr2;
10877 }
10878
10879 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
10880 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10881 return VINF_EM_RAW_INJECT_TRPM_EVENT;
10882 }
10883 }
10884
10885 /** @todo Emulate task switch someday, currently just going back to ring-3 for
10886 * emulation. */
10887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
10888 return VERR_EM_INTERPRETER;
10889}
10890
10891
10892/**
10893 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
10894 */
10895HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10896{
10897 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10898 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
10899 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
10900 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
10901 AssertRCReturn(rc, rc);
10902 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
10903 return VINF_EM_DBG_STEPPED;
10904}
10905
10906
10907/**
10908 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
10909 */
10910HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10911{
10912 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10913
10914 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10915 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10916 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
10917 return VINF_SUCCESS;
10918 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
10919 return rc;
10920
10921#if 0
10922 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
10923 * just sync the whole thing. */
10924 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10925#else
10926 /* Aggressive state sync. for now. */
10927 rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
10928 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
10929 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10930#endif
10931 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10932 AssertRCReturn(rc, rc);
10933
10934 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
10935 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
10936 switch (uAccessType)
10937 {
10938 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
10939 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
10940 {
10941 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
10942 && VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) == 0x80)
10943 {
10944 AssertMsgFailed(("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
10945 }
10946
10947 RTGCPHYS GCPhys = pMixedCtx->msrApicBase; /* Always up-to-date, msrApicBase is not part of the VMCS. */
10948 GCPhys &= PAGE_BASE_GC_MASK;
10949 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
10950 PVM pVM = pVCpu->CTX_SUFF(pVM);
10951 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGv Off=%#x\n", uAccessType, GCPhys,
10952 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
10953
10954 VBOXSTRICTRC rc2 = IOMMMIOPhysHandler(pVM, pVCpu,
10955 (uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ) ? 0 : X86_TRAP_PF_RW,
10956 CPUMCTX2CORE(pMixedCtx), GCPhys);
10957 rc = VBOXSTRICTRC_VAL(rc2);
10958 Log4(("ApicAccess rc=%d\n", rc));
10959 if ( rc == VINF_SUCCESS
10960 || rc == VERR_PAGE_TABLE_NOT_PRESENT
10961 || rc == VERR_PAGE_NOT_PRESENT)
10962 {
10963 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
10964 | HM_CHANGED_GUEST_RSP
10965 | HM_CHANGED_GUEST_RFLAGS
10966 | HM_CHANGED_VMX_GUEST_APIC_STATE);
10967 rc = VINF_SUCCESS;
10968 }
10969 break;
10970 }
10971
10972 default:
10973 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
10974 rc = VINF_EM_RAW_EMULATE_INSTR;
10975 break;
10976 }
10977
10978 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
10979 return rc;
10980}
10981
10982
10983/**
10984 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
10985 * VM-exit.
10986 */
10987HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10988{
10989 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10990
10991 /* We should -not- get this VM-exit if the guest's debug registers were active. */
10992 if (pVmxTransient->fWasGuestDebugStateActive)
10993 {
10994 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
10995 HMVMX_RETURN_UNEXPECTED_EXIT();
10996 }
10997
10998 int rc = VERR_INTERNAL_ERROR_5;
10999 if ( !DBGFIsStepping(pVCpu)
11000 && !pVCpu->hm.s.fSingleInstruction
11001 && !pVmxTransient->fWasHyperDebugStateActive)
11002 {
11003 /* Don't intercept MOV DRx and #DB any more. */
11004 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
11005 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
11006 AssertRCReturn(rc, rc);
11007
11008 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11009 {
11010#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11011 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_DB);
11012 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
11013 AssertRCReturn(rc, rc);
11014#endif
11015 }
11016
11017 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
11018 VMMRZCallRing3Disable(pVCpu);
11019 HM_DISABLE_PREEMPT_IF_NEEDED();
11020
11021 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
11022 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
11023 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
11024
11025 HM_RESTORE_PREEMPT_IF_NEEDED();
11026 VMMRZCallRing3Enable(pVCpu);
11027
11028#ifdef VBOX_WITH_STATISTICS
11029 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11030 AssertRCReturn(rc, rc);
11031 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11032 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11033 else
11034 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11035#endif
11036 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
11037 return VINF_SUCCESS;
11038 }
11039
11040 /*
11041 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
11042 * Update the segment registers and DR7 from the CPU.
11043 */
11044 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11045 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11046 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11047 AssertRCReturn(rc, rc);
11048 Log4(("CS:RIP=%04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11049
11050 PVM pVM = pVCpu->CTX_SUFF(pVM);
11051 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
11052 {
11053 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11054 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
11055 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
11056 if (RT_SUCCESS(rc))
11057 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
11058 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
11059 }
11060 else
11061 {
11062 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
11063 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
11064 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
11065 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
11066 }
11067
11068 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
11069 if (RT_SUCCESS(rc))
11070 {
11071 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11072 AssertRCReturn(rc2, rc2);
11073 }
11074 return rc;
11075}
11076
11077
11078/**
11079 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
11080 * Conditional VM-exit.
11081 */
11082HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11083{
11084 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11085 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11086
11087 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11088 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11089 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11090 return VINF_SUCCESS;
11091 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11092 return rc;
11093
11094 RTGCPHYS GCPhys = 0;
11095 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11096
11097#if 0
11098 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11099#else
11100 /* Aggressive state sync. for now. */
11101 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11102 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11103 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11104#endif
11105 AssertRCReturn(rc, rc);
11106
11107 /*
11108 * If we succeed, resume guest execution.
11109 * If we fail in interpreting the instruction because we couldn't get the guest physical address
11110 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
11111 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
11112 * weird case. See @bugref{6043}.
11113 */
11114 PVM pVM = pVCpu->CTX_SUFF(pVM);
11115 VBOXSTRICTRC rc2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
11116 rc = VBOXSTRICTRC_VAL(rc2);
11117 Log4(("EPT misconfig at %#RGv RIP=%#RX64 rc=%d\n", GCPhys, pMixedCtx->rip, rc));
11118 if ( rc == VINF_SUCCESS
11119 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11120 || rc == VERR_PAGE_NOT_PRESENT)
11121 {
11122 /* Successfully handled MMIO operation. */
11123 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11124 | HM_CHANGED_GUEST_RSP
11125 | HM_CHANGED_GUEST_RFLAGS
11126 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11127 rc = VINF_SUCCESS;
11128 }
11129 return rc;
11130}
11131
11132
11133/**
11134 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
11135 * VM-exit.
11136 */
11137HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11138{
11139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11140 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
11141
11142 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11143 int rc = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11144 if (RT_UNLIKELY(rc == VINF_HM_DOUBLE_FAULT))
11145 return VINF_SUCCESS;
11146 else if (RT_UNLIKELY(rc == VINF_EM_RESET))
11147 return rc;
11148
11149 RTGCPHYS GCPhys = 0;
11150 rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
11151 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11152#if 0
11153 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
11154#else
11155 /* Aggressive state sync. for now. */
11156 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
11157 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11158 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11159#endif
11160 AssertRCReturn(rc, rc);
11161
11162 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
11163 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
11164
11165 RTGCUINT uErrorCode = 0;
11166 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
11167 uErrorCode |= X86_TRAP_PF_ID;
11168 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
11169 uErrorCode |= X86_TRAP_PF_RW;
11170 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
11171 uErrorCode |= X86_TRAP_PF_P;
11172
11173 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
11174
11175 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:EIP=%04x:%#RX64\n", pVmxTransient->uExitQualification, GCPhys,
11176 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11177
11178 /* Handle the pagefault trap for the nested shadow table. */
11179 PVM pVM = pVCpu->CTX_SUFF(pVM);
11180 rc = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
11181 TRPMResetTrap(pVCpu);
11182
11183 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
11184 if ( rc == VINF_SUCCESS
11185 || rc == VERR_PAGE_TABLE_NOT_PRESENT
11186 || rc == VERR_PAGE_NOT_PRESENT)
11187 {
11188 /* Successfully synced our nested page tables. */
11189 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
11190 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11191 | HM_CHANGED_GUEST_RSP
11192 | HM_CHANGED_GUEST_RFLAGS);
11193 return VINF_SUCCESS;
11194 }
11195
11196 Log4(("EPT return to ring-3 rc=%Rrc\n", rc));
11197 return rc;
11198}
11199
11200/** @} */
11201
11202/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11203/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
11204/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11205
11206/** @name VM-exit exception handlers.
11207 * @{
11208 */
11209
11210/**
11211 * VM-exit exception handler for #MF (Math Fault: floating point exception).
11212 */
11213static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11214{
11215 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11216 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
11217
11218 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11219 AssertRCReturn(rc, rc);
11220
11221 if (!(pMixedCtx->cr0 & X86_CR0_NE))
11222 {
11223 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
11224 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
11225
11226 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
11227 * provides VM-exit instruction length. If this causes problem later,
11228 * disassemble the instruction like it's done on AMD-V. */
11229 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11230 AssertRCReturn(rc2, rc2);
11231 return rc;
11232 }
11233
11234 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11235 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11236 return rc;
11237}
11238
11239
11240/**
11241 * VM-exit exception handler for #BP (Breakpoint exception).
11242 */
11243static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11244{
11245 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11246 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
11247
11248 /** @todo Try optimize this by not saving the entire guest state unless
11249 * really needed. */
11250 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11251 AssertRCReturn(rc, rc);
11252
11253 PVM pVM = pVCpu->CTX_SUFF(pVM);
11254 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11255 if (rc == VINF_EM_RAW_GUEST_TRAP)
11256 {
11257 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11258 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11259 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11260 AssertRCReturn(rc, rc);
11261
11262 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11263 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11264 }
11265
11266 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
11267 return rc;
11268}
11269
11270
11271/**
11272 * VM-exit exception handler for #DB (Debug exception).
11273 */
11274static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11275{
11276 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
11278 Log6(("XcptDB\n"));
11279
11280 /*
11281 * Get the DR6-like values from the exit qualification and pass it to DBGF
11282 * for processing.
11283 */
11284 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11285 AssertRCReturn(rc, rc);
11286
11287 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
11288 uint64_t uDR6 = X86_DR6_INIT_VAL;
11289 uDR6 |= ( pVmxTransient->uExitQualification
11290 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
11291
11292 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
11293 if (rc == VINF_EM_RAW_GUEST_TRAP)
11294 {
11295 /*
11296 * The exception was for the guest. Update DR6, DR7.GD and
11297 * IA32_DEBUGCTL.LBR before forwarding it.
11298 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
11299 */
11300 VMMRZCallRing3Disable(pVCpu);
11301 HM_DISABLE_PREEMPT_IF_NEEDED();
11302
11303 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
11304 pMixedCtx->dr[6] |= uDR6;
11305 if (CPUMIsGuestDebugStateActive(pVCpu))
11306 ASMSetDR6(pMixedCtx->dr[6]);
11307
11308 HM_RESTORE_PREEMPT_IF_NEEDED();
11309 VMMRZCallRing3Enable(pVCpu);
11310
11311 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
11312 AssertRCReturn(rc, rc);
11313
11314 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
11315 pMixedCtx->dr[7] &= ~X86_DR7_GD;
11316
11317 /* Paranoia. */
11318 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
11319 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
11320
11321 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
11322 AssertRCReturn(rc, rc);
11323
11324 /*
11325 * Raise #DB in the guest.
11326 *
11327 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
11328 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP and not the 'normal' #DB.
11329 * Thus it -may- trigger different handling in the CPU (like skipped DPL checks). See @bugref{6398}.
11330 *
11331 * Since ICEBP isn't documented on Intel, see AMD spec. 15.20 "Event Injection".
11332 */
11333 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11334 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11335 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11336 AssertRCReturn(rc, rc);
11337 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11338 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11339 return VINF_SUCCESS;
11340 }
11341
11342 /*
11343 * Not a guest trap, must be a hypervisor related debug event then.
11344 * Update DR6 in case someone is interested in it.
11345 */
11346 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
11347 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
11348 CPUMSetHyperDR6(pVCpu, uDR6);
11349
11350 return rc;
11351}
11352
11353
11354/**
11355 * VM-exit exception handler for #NM (Device-not-available exception: floating
11356 * point exception).
11357 */
11358static int hmR0VmxExitXcptNM(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11359{
11360 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11361
11362 /* We require CR0 and EFER. EFER is always up-to-date. */
11363 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11364 AssertRCReturn(rc, rc);
11365
11366 /* We're playing with the host CPU state here, have to disable preemption or longjmp. */
11367 VMMRZCallRing3Disable(pVCpu);
11368 HM_DISABLE_PREEMPT_IF_NEEDED();
11369
11370 /* If the guest FPU was active at the time of the #NM exit, then it's a guest fault. */
11371 if (pVmxTransient->fWasGuestFPUStateActive)
11372 {
11373 rc = VINF_EM_RAW_GUEST_TRAP;
11374 Assert(CPUMIsGuestFPUStateActive(pVCpu) || HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
11375 }
11376 else
11377 {
11378#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11379 Assert(!pVmxTransient->fWasGuestFPUStateActive);
11380#endif
11381 rc = CPUMR0Trap07Handler(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
11382 Assert(rc == VINF_EM_RAW_GUEST_TRAP || (rc == VINF_SUCCESS && CPUMIsGuestFPUStateActive(pVCpu)));
11383 }
11384
11385 HM_RESTORE_PREEMPT_IF_NEEDED();
11386 VMMRZCallRing3Enable(pVCpu);
11387
11388 if (rc == VINF_SUCCESS)
11389 {
11390 /* Guest FPU state was activated, we'll want to change CR0 FPU intercepts before the next VM-reentry. */
11391 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
11392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowNM);
11393 pVCpu->hm.s.fUseGuestFpu = true;
11394 }
11395 else
11396 {
11397 /* Forward #NM to the guest. */
11398 Assert(rc == VINF_EM_RAW_GUEST_TRAP);
11399 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11400 AssertRCReturn(rc, rc);
11401 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11402 pVmxTransient->cbInstr, 0 /* error code */, 0 /* GCPtrFaultAddress */);
11403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11404 }
11405
11406 return VINF_SUCCESS;
11407}
11408
11409
11410/**
11411 * VM-exit exception handler for #GP (General-protection exception).
11412 *
11413 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
11414 */
11415static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11416{
11417 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
11419
11420 int rc = VERR_INTERNAL_ERROR_5;
11421 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11422 {
11423#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11424 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
11425 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11426 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11427 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11428 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11429 AssertRCReturn(rc, rc);
11430 Log4(("#GP Gst: CS:RIP %04x:%#RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
11431 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
11432 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11433 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11434 return rc;
11435#else
11436 /* We don't intercept #GP. */
11437 AssertMsgFailed(("Unexpected VM-exit caused by #GP exception\n"));
11438 NOREF(pVmxTransient);
11439 return VERR_VMX_UNEXPECTED_EXCEPTION;
11440#endif
11441 }
11442
11443 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11444 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
11445
11446 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
11447 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11448 AssertRCReturn(rc, rc);
11449
11450 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
11451 uint32_t cbOp = 0;
11452 PVM pVM = pVCpu->CTX_SUFF(pVM);
11453 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
11454 if (RT_SUCCESS(rc))
11455 {
11456 rc = VINF_SUCCESS;
11457 Assert(cbOp == pDis->cbInstr);
11458 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%#RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
11459 switch (pDis->pCurInstr->uOpcode)
11460 {
11461 case OP_CLI:
11462 {
11463 pMixedCtx->eflags.Bits.u1IF = 0;
11464 pMixedCtx->rip += pDis->cbInstr;
11465 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11466 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
11468 break;
11469 }
11470
11471 case OP_STI:
11472 {
11473 pMixedCtx->eflags.Bits.u1IF = 1;
11474 pMixedCtx->rip += pDis->cbInstr;
11475 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
11476 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
11477 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11478 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
11480 break;
11481 }
11482
11483 case OP_HLT:
11484 {
11485 rc = VINF_EM_HALT;
11486 pMixedCtx->rip += pDis->cbInstr;
11487 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
11488 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11489 break;
11490 }
11491
11492 case OP_POPF:
11493 {
11494 Log4(("POPF CS:RIP %04x:%#RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
11495 uint32_t cbParm;
11496 uint32_t uMask;
11497 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11498 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11499 {
11500 cbParm = 4;
11501 uMask = 0xffffffff;
11502 }
11503 else
11504 {
11505 cbParm = 2;
11506 uMask = 0xffff;
11507 }
11508
11509 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
11510 RTGCPTR GCPtrStack = 0;
11511 X86EFLAGS Eflags;
11512 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11513 &GCPtrStack);
11514 if (RT_SUCCESS(rc))
11515 {
11516 Assert(sizeof(Eflags.u32) >= cbParm);
11517 Eflags.u32 = 0;
11518 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm);
11519 }
11520 if (RT_FAILURE(rc))
11521 {
11522 rc = VERR_EM_INTERPRETER;
11523 break;
11524 }
11525 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
11526 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11527 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
11528 pMixedCtx->eflags.Bits.u1RF = 0; /* The RF bit is always cleared by POPF; see Intel Instruction reference. */
11529 pMixedCtx->esp += cbParm;
11530 pMixedCtx->esp &= uMask;
11531 pMixedCtx->rip += pDis->cbInstr;
11532 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11533 | HM_CHANGED_GUEST_RSP
11534 | HM_CHANGED_GUEST_RFLAGS);
11535 /* Generate a pending-debug exception when stepping over POPF regardless of how POPF modifies EFLAGS.TF. */
11536 if (fStepping)
11537 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11538
11539 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
11540 break;
11541 }
11542
11543 case OP_PUSHF:
11544 {
11545 uint32_t cbParm;
11546 uint32_t uMask;
11547 if (pDis->fPrefix & DISPREFIX_OPSIZE)
11548 {
11549 cbParm = 4;
11550 uMask = 0xffffffff;
11551 }
11552 else
11553 {
11554 cbParm = 2;
11555 uMask = 0xffff;
11556 }
11557
11558 /* Get the stack pointer & push the contents of eflags onto the stack. */
11559 RTGCPTR GCPtrStack = 0;
11560 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
11561 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
11562 if (RT_FAILURE(rc))
11563 {
11564 rc = VERR_EM_INTERPRETER;
11565 break;
11566 }
11567 X86EFLAGS Eflags = pMixedCtx->eflags;
11568 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
11569 Eflags.Bits.u1RF = 0;
11570 Eflags.Bits.u1VM = 0;
11571
11572 rc = PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm);
11573 if (RT_FAILURE(rc))
11574 {
11575 rc = VERR_EM_INTERPRETER;
11576 break;
11577 }
11578 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
11579 pMixedCtx->esp -= cbParm;
11580 pMixedCtx->esp &= uMask;
11581 pMixedCtx->rip += pDis->cbInstr;
11582 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP);
11583 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11584 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
11585 break;
11586 }
11587
11588 case OP_IRET:
11589 {
11590 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
11591 * instruction reference. */
11592 RTGCPTR GCPtrStack = 0;
11593 uint32_t uMask = 0xffff;
11594 bool fStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
11595 uint16_t aIretFrame[3];
11596 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
11597 {
11598 rc = VERR_EM_INTERPRETER;
11599 break;
11600 }
11601 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
11602 &GCPtrStack);
11603 if (RT_SUCCESS(rc))
11604 rc = PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame));
11605 if (RT_FAILURE(rc))
11606 {
11607 rc = VERR_EM_INTERPRETER;
11608 break;
11609 }
11610 pMixedCtx->eip = 0;
11611 pMixedCtx->ip = aIretFrame[0];
11612 pMixedCtx->cs.Sel = aIretFrame[1];
11613 pMixedCtx->cs.ValidSel = aIretFrame[1];
11614 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
11615 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~(X86_EFL_POPF_BITS & uMask))
11616 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
11617 pMixedCtx->sp += sizeof(aIretFrame);
11618 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11619 | HM_CHANGED_GUEST_SEGMENT_REGS
11620 | HM_CHANGED_GUEST_RSP
11621 | HM_CHANGED_GUEST_RFLAGS);
11622 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
11623 if (fStepping)
11624 hmR0VmxSetPendingDebugXcpt(pVCpu, pMixedCtx);
11625 Log4(("IRET %#RX32 to %04x:%x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
11626 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
11627 break;
11628 }
11629
11630 case OP_INT:
11631 {
11632 uint16_t uVector = pDis->Param1.uValue & 0xff;
11633 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
11634 /* INT clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11636 break;
11637 }
11638
11639 case OP_INTO:
11640 {
11641 if (pMixedCtx->eflags.Bits.u1OF)
11642 {
11643 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
11644 /* INTO clears EFLAGS.TF, we mustn't set any pending debug exceptions here. */
11645 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
11646 }
11647 break;
11648 }
11649
11650 default:
11651 {
11652 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
11653 EMCODETYPE_SUPERVISOR);
11654 rc = VBOXSTRICTRC_VAL(rc2);
11655 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
11656 /** @todo We have to set pending-debug exceptions here when the guest is
11657 * single-stepping depending on the instruction that was interpreted. */
11658 Log4(("#GP rc=%Rrc\n", rc));
11659 break;
11660 }
11661 }
11662 }
11663 else
11664 rc = VERR_EM_INTERPRETER;
11665
11666 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
11667 ("#GP Unexpected rc=%Rrc\n", rc));
11668 return rc;
11669}
11670
11671
11672#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
11673/**
11674 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
11675 * the exception reported in the VMX transient structure back into the VM.
11676 *
11677 * @remarks Requires uExitIntInfo in the VMX transient structure to be
11678 * up-to-date.
11679 */
11680static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11681{
11682 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11683
11684 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
11685 hmR0VmxCheckExitDueToEventDelivery(). */
11686 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11687 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11688 AssertRCReturn(rc, rc);
11689 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
11690
11691#ifdef DEBUG_ramshankar
11692 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11693 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
11694 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
11695#endif
11696
11697 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11698 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
11699 return VINF_SUCCESS;
11700}
11701#endif
11702
11703
11704/**
11705 * VM-exit exception handler for #PF (Page-fault exception).
11706 */
11707static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11708{
11709 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
11710 PVM pVM = pVCpu->CTX_SUFF(pVM);
11711 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11712 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11713 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11714 AssertRCReturn(rc, rc);
11715
11716#if defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) || defined(HMVMX_ALWAYS_TRAP_PF)
11717 if (pVM->hm.s.fNestedPaging)
11718 {
11719 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
11720 if (RT_LIKELY(!pVmxTransient->fVectoringPF))
11721 {
11722 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11723 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11724 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
11725 }
11726 else
11727 {
11728 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11729 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11730 Log4(("Pending #DF due to vectoring #PF. NP\n"));
11731 }
11732 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11733 return rc;
11734 }
11735#else
11736 Assert(!pVM->hm.s.fNestedPaging);
11737 NOREF(pVM);
11738#endif
11739
11740 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11741 AssertRCReturn(rc, rc);
11742
11743 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
11744 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
11745
11746 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
11747 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
11748 (RTGCPTR)pVmxTransient->uExitQualification);
11749
11750 Log4(("#PF: rc=%Rrc\n", rc));
11751 if (rc == VINF_SUCCESS)
11752 {
11753 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
11754 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
11755 * memory? We don't update the whole state here... */
11756 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
11757 | HM_CHANGED_GUEST_RSP
11758 | HM_CHANGED_GUEST_RFLAGS
11759 | HM_CHANGED_VMX_GUEST_APIC_STATE);
11760 TRPMResetTrap(pVCpu);
11761 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
11762 return rc;
11763 }
11764 else if (rc == VINF_EM_RAW_GUEST_TRAP)
11765 {
11766 if (!pVmxTransient->fVectoringPF)
11767 {
11768 /* It's a guest page fault and needs to be reflected to the guest. */
11769 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
11770 TRPMResetTrap(pVCpu);
11771 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
11772 pMixedCtx->cr2 = pVmxTransient->uExitQualification; /* Update here in case we go back to ring-3 before injection. */
11773 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
11774 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
11775 }
11776 else
11777 {
11778 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
11779 TRPMResetTrap(pVCpu);
11780 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
11781 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
11782 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
11783 }
11784
11785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
11786 return VINF_SUCCESS;
11787 }
11788
11789 TRPMResetTrap(pVCpu);
11790 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
11791 return rc;
11792}
11793
11794/** @} */
11795
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette