VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72600

Last change on this file since 72600 was 72600, checked in by vboxsync, 6 years ago

VMM: Eliminated VBOX_WITH_2ND_IEM_STEP.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 596.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72600 2018-06-18 13:40:48Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#define HMVMX_USE_IEM_EVENT_REFLECTION
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DR7 \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
142 * due to bugs in Intel CPUs.
143 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
144 * support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/** Helper macro for VM-exit handlers called unexpectedly. */
187#define HMVMX_RETURN_UNEXPECTED_EXIT() \
188 do { \
189 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
190 return VERR_VMX_UNEXPECTED_EXIT; \
191 } while (0)
192
193
194/*********************************************************************************************************************************
195* Structures and Typedefs *
196*********************************************************************************************************************************/
197/**
198 * VMX transient state.
199 *
200 * A state structure for holding miscellaneous information across
201 * VMX non-root operation and restored after the transition.
202 */
203typedef struct VMXTRANSIENT
204{
205 /** The host's rflags/eflags. */
206 RTCCUINTREG fEFlags;
207#if HC_ARCH_BITS == 32
208 uint32_t u32Alignment0;
209#endif
210 /** The guest's TPR value used for TPR shadowing. */
211 uint8_t u8GuestTpr;
212 /** Alignment. */
213 uint8_t abAlignment0[7];
214
215 /** The basic VM-exit reason. */
216 uint16_t uExitReason;
217 /** Alignment. */
218 uint16_t u16Alignment0;
219 /** The VM-exit interruption error code. */
220 uint32_t uExitIntErrorCode;
221 /** The VM-exit exit code qualification. */
222 uint64_t uExitQualification;
223
224 /** The VM-exit interruption-information field. */
225 uint32_t uExitIntInfo;
226 /** The VM-exit instruction-length field. */
227 uint32_t cbInstr;
228 /** The VM-exit instruction-information field. */
229 union
230 {
231 /** Plain unsigned int representation. */
232 uint32_t u;
233 /** INS and OUTS information. */
234 struct
235 {
236 uint32_t u7Reserved0 : 7;
237 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
238 uint32_t u3AddrSize : 3;
239 uint32_t u5Reserved1 : 5;
240 /** The segment register (X86_SREG_XXX). */
241 uint32_t iSegReg : 3;
242 uint32_t uReserved2 : 14;
243 } StrIo;
244 /** INVEPT, INVVPID, INVPCID information. */
245 struct
246 {
247 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
248 uint32_t u2Scaling : 2;
249 uint32_t u5Reserved0 : 5;
250 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
251 uint32_t u3AddrSize : 3;
252 uint32_t u1Reserved0 : 1;
253 uint32_t u4Reserved0 : 4;
254 /** The segment register (X86_SREG_XXX). */
255 uint32_t iSegReg : 3;
256 /** The index register (X86_GREG_XXX). */
257 uint32_t iIdxReg : 4;
258 /** Set if index register is invalid. */
259 uint32_t fIdxRegValid : 1;
260 /** The base register (X86_GREG_XXX). */
261 uint32_t iBaseReg : 4;
262 /** Set if base register is invalid. */
263 uint32_t fBaseRegValid : 1;
264 /** Register 2 (X86_GREG_XXX). */
265 uint32_t iReg2 : 4;
266 } Inv;
267 } ExitInstrInfo;
268 /** Whether the VM-entry failed or not. */
269 bool fVMEntryFailed;
270 /** Alignment. */
271 uint8_t abAlignment1[3];
272
273 /** The VM-entry interruption-information field. */
274 uint32_t uEntryIntInfo;
275 /** The VM-entry exception error code field. */
276 uint32_t uEntryXcptErrorCode;
277 /** The VM-entry instruction length field. */
278 uint32_t cbEntryInstr;
279
280 /** IDT-vectoring information field. */
281 uint32_t uIdtVectoringInfo;
282 /** IDT-vectoring error code. */
283 uint32_t uIdtVectoringErrorCode;
284
285 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
286 uint32_t fVmcsFieldsRead;
287
288 /** Whether the guest debug state was active at the time of VM-exit. */
289 bool fWasGuestDebugStateActive;
290 /** Whether the hyper debug state was active at the time of VM-exit. */
291 bool fWasHyperDebugStateActive;
292 /** Whether TSC-offsetting should be setup before VM-entry. */
293 bool fUpdateTscOffsettingAndPreemptTimer;
294 /** Whether the VM-exit was caused by a page-fault during delivery of a
295 * contributory exception or a page-fault. */
296 bool fVectoringDoublePF;
297 /** Whether the VM-exit was caused by a page-fault during delivery of an
298 * external interrupt or NMI. */
299 bool fVectoringPF;
300} VMXTRANSIENT;
301AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
305AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
306/** Pointer to VMX transient state. */
307typedef VMXTRANSIENT *PVMXTRANSIENT;
308
309
310/**
311 * MSR-bitmap read permissions.
312 */
313typedef enum VMXMSREXITREAD
314{
315 /** Reading this MSR causes a VM-exit. */
316 VMXMSREXIT_INTERCEPT_READ = 0xb,
317 /** Reading this MSR does not cause a VM-exit. */
318 VMXMSREXIT_PASSTHRU_READ
319} VMXMSREXITREAD;
320/** Pointer to MSR-bitmap read permissions. */
321typedef VMXMSREXITREAD* PVMXMSREXITREAD;
322
323/**
324 * MSR-bitmap write permissions.
325 */
326typedef enum VMXMSREXITWRITE
327{
328 /** Writing to this MSR causes a VM-exit. */
329 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
330 /** Writing to this MSR does not cause a VM-exit. */
331 VMXMSREXIT_PASSTHRU_WRITE
332} VMXMSREXITWRITE;
333/** Pointer to MSR-bitmap write permissions. */
334typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
335
336
337/**
338 * VMX VM-exit handler.
339 *
340 * @returns Strict VBox status code (i.e. informational status codes too).
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 */
347#ifndef HMVMX_USE_FUNCTION_TABLE
348typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
349#else
350typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
351/** Pointer to VM-exit handler. */
352typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
353#endif
354
355/**
356 * VMX VM-exit handler, non-strict status code.
357 *
358 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
359 *
360 * @returns VBox status code, no informational status code returned.
361 * @param pVCpu The cross context virtual CPU structure.
362 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
363 * out-of-sync. Make sure to update the required
364 * fields before using them.
365 * @param pVmxTransient Pointer to the VMX-transient structure.
366 *
367 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
368 * use of that status code will be replaced with VINF_EM_SOMETHING
369 * later when switching over to IEM.
370 */
371#ifndef HMVMX_USE_FUNCTION_TABLE
372typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
373#else
374typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
375#endif
376
377
378/*********************************************************************************************************************************
379* Internal Functions *
380*********************************************************************************************************************************/
381static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
382static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
383static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
384static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
385 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
386 bool fStepping, uint32_t *puIntState);
387#if HC_ARCH_BITS == 32
388static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
389#endif
390#ifndef HMVMX_USE_FUNCTION_TABLE
391DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
392# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
393# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
394#else
395# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
396# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
397#endif
398
399
400/** @name VM-exit handlers.
401 * @{
402 */
403static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
404static FNVMXEXITHANDLER hmR0VmxExitExtInt;
405static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
412static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
413static FNVMXEXITHANDLER hmR0VmxExitCpuid;
414static FNVMXEXITHANDLER hmR0VmxExitGetsec;
415static FNVMXEXITHANDLER hmR0VmxExitHlt;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
417static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
418static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
419static FNVMXEXITHANDLER hmR0VmxExitVmcall;
420static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
423static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
424static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
425static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
426static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
427static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
431static FNVMXEXITHANDLER hmR0VmxExitMwait;
432static FNVMXEXITHANDLER hmR0VmxExitMtf;
433static FNVMXEXITHANDLER hmR0VmxExitMonitor;
434static FNVMXEXITHANDLER hmR0VmxExitPause;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
437static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
438static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
439static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
440static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
441static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
442static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
443static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
444static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
445static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
446static FNVMXEXITHANDLER hmR0VmxExitRdrand;
447static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
448/** @} */
449
450static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
451static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
452static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
453static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
454static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
458
459
460/*********************************************************************************************************************************
461* Global Variables *
462*********************************************************************************************************************************/
463#ifdef HMVMX_USE_FUNCTION_TABLE
464
465/**
466 * VMX_EXIT dispatch table.
467 */
468static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
469{
470 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
471 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
472 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
473 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
474 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
475 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
476 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
477 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
478 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
479 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
480 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
481 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
482 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
483 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
484 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
485 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
486 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
487 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
488 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
489 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
490 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
491 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
492 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
493 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
494 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
495 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
496 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
497 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
498 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
499 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
500 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
501 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
502 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
503 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
504 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
505 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
506 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
507 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
508 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
510 /* 40 UNDEFINED */ hmR0VmxExitPause,
511 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
512 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
513 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
514 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
515 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
516 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
517 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
518 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
519 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
520 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
521 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
522 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
523 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
524 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
525 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
526 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
527 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
528 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
529 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
530 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
531 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
532 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
533 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
534 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
535};
536#endif /* HMVMX_USE_FUNCTION_TABLE */
537
538#ifdef VBOX_STRICT
539static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
540{
541 /* 0 */ "(Not Used)",
542 /* 1 */ "VMCALL executed in VMX root operation.",
543 /* 2 */ "VMCLEAR with invalid physical address.",
544 /* 3 */ "VMCLEAR with VMXON pointer.",
545 /* 4 */ "VMLAUNCH with non-clear VMCS.",
546 /* 5 */ "VMRESUME with non-launched VMCS.",
547 /* 6 */ "VMRESUME after VMXOFF",
548 /* 7 */ "VM-entry with invalid control fields.",
549 /* 8 */ "VM-entry with invalid host state fields.",
550 /* 9 */ "VMPTRLD with invalid physical address.",
551 /* 10 */ "VMPTRLD with VMXON pointer.",
552 /* 11 */ "VMPTRLD with incorrect revision identifier.",
553 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
554 /* 13 */ "VMWRITE to read-only VMCS component.",
555 /* 14 */ "(Not Used)",
556 /* 15 */ "VMXON executed in VMX root operation.",
557 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
558 /* 17 */ "VM-entry with non-launched executing VMCS.",
559 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
560 /* 19 */ "VMCALL with non-clear VMCS.",
561 /* 20 */ "VMCALL with invalid VM-exit control fields.",
562 /* 21 */ "(Not Used)",
563 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
564 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
565 /* 24 */ "VMCALL with invalid SMM-monitor features.",
566 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
567 /* 26 */ "VM-entry with events blocked by MOV SS.",
568 /* 27 */ "(Not Used)",
569 /* 28 */ "Invalid operand to INVEPT/INVVPID."
570};
571#endif /* VBOX_STRICT */
572
573
574
575/**
576 * Updates the VM's last error record.
577 *
578 * If there was a VMX instruction error, reads the error data from the VMCS and
579 * updates VCPU's last error record as well.
580 *
581 * @param pVM The cross context VM structure.
582 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
583 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
584 * VERR_VMX_INVALID_VMCS_FIELD.
585 * @param rc The error code.
586 */
587static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
588{
589 AssertPtr(pVM);
590 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591 || rc == VERR_VMX_UNABLE_TO_START_VM)
592 {
593 AssertPtrReturnVoid(pVCpu);
594 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595 }
596 pVM->hm.s.lLastError = rc;
597}
598
599
600/**
601 * Reads the VM-entry interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636#ifdef VBOX_STRICT
637/**
638 * Reads the VM-entry exception error code field from the VMCS into
639 * the VMX transient structure.
640 *
641 * @returns VBox status code.
642 * @param pVmxTransient Pointer to the VMX transient structure.
643 *
644 * @remarks No-long-jump zone!!!
645 */
646DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
647{
648 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
649 AssertRCReturn(rc, rc);
650 return VINF_SUCCESS;
651}
652#endif /* VBOX_STRICT */
653
654
655/**
656 * Reads the VM-exit interruption-information field from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit interruption error code from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction length field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the VM-exit instruction-information field from the VMCS into
714 * the VMX transient structure.
715 *
716 * @returns VBox status code.
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
722 {
723 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the exit code qualification from the VMCS into the VMX transient
733 * structure.
734 *
735 * @returns VBox status code.
736 * @param pVCpu The cross context virtual CPU structure of the
737 * calling EMT. (Required for the VMCS cache case.)
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 */
740DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
743 {
744 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring information field from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 *
759 * @remarks No-long-jump zone!!!
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Reads the IDT-vectoring error code from the VMCS into the VMX
775 * transient structure.
776 *
777 * @returns VBox status code.
778 * @param pVmxTransient Pointer to the VMX transient structure.
779 */
780DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
781{
782 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
783 {
784 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
785 AssertRCReturn(rc, rc);
786 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
787 }
788 return VINF_SUCCESS;
789}
790
791
792/**
793 * Enters VMX root mode operation on the current CPU.
794 *
795 * @returns VBox status code.
796 * @param pVM The cross context VM structure. Can be
797 * NULL, after a resume.
798 * @param HCPhysCpuPage Physical address of the VMXON region.
799 * @param pvCpuPage Pointer to the VMXON region.
800 */
801static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
802{
803 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
804 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
805 Assert(pvCpuPage);
806 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
807
808 if (pVM)
809 {
810 /* Write the VMCS revision dword to the VMXON region. */
811 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
812 }
813
814 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
815 RTCCUINTREG fEFlags = ASMIntDisableFlags();
816
817 /* Enable the VMX bit in CR4 if necessary. */
818 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
819
820 /* Enter VMX root mode. */
821 int rc = VMXEnable(HCPhysCpuPage);
822 if (RT_FAILURE(rc))
823 {
824 if (!(uOldCr4 & X86_CR4_VMXE))
825 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
826
827 if (pVM)
828 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
829 }
830
831 /* Restore interrupts. */
832 ASMSetFlags(fEFlags);
833 return rc;
834}
835
836
837/**
838 * Exits VMX root mode operation on the current CPU.
839 *
840 * @returns VBox status code.
841 */
842static int hmR0VmxLeaveRootMode(void)
843{
844 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
845
846 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
847 RTCCUINTREG fEFlags = ASMIntDisableFlags();
848
849 /* If we're for some reason not in VMX root mode, then don't leave it. */
850 RTCCUINTREG uHostCR4 = ASMGetCR4();
851
852 int rc;
853 if (uHostCR4 & X86_CR4_VMXE)
854 {
855 /* Exit VMX root mode and clear the VMX bit in CR4. */
856 VMXDisable();
857 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
858 rc = VINF_SUCCESS;
859 }
860 else
861 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
862
863 /* Restore interrupts. */
864 ASMSetFlags(fEFlags);
865 return rc;
866}
867
868
869/**
870 * Allocates and maps one physically contiguous page. The allocated page is
871 * zero'd out. (Used by various VT-x structures).
872 *
873 * @returns IPRT status code.
874 * @param pMemObj Pointer to the ring-0 memory object.
875 * @param ppVirt Where to store the virtual address of the
876 * allocation.
877 * @param pHCPhys Where to store the physical address of the
878 * allocation.
879 */
880DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
881{
882 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
883 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
884 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
885
886 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
887 if (RT_FAILURE(rc))
888 return rc;
889 *ppVirt = RTR0MemObjAddress(*pMemObj);
890 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
891 ASMMemZero32(*ppVirt, PAGE_SIZE);
892 return VINF_SUCCESS;
893}
894
895
896/**
897 * Frees and unmaps an allocated physical page.
898 *
899 * @param pMemObj Pointer to the ring-0 memory object.
900 * @param ppVirt Where to re-initialize the virtual address of
901 * allocation as 0.
902 * @param pHCPhys Where to re-initialize the physical address of the
903 * allocation as 0.
904 */
905DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
906{
907 AssertPtr(pMemObj);
908 AssertPtr(ppVirt);
909 AssertPtr(pHCPhys);
910 if (*pMemObj != NIL_RTR0MEMOBJ)
911 {
912 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
913 AssertRC(rc);
914 *pMemObj = NIL_RTR0MEMOBJ;
915 *ppVirt = 0;
916 *pHCPhys = 0;
917 }
918}
919
920
921/**
922 * Worker function to free VT-x related structures.
923 *
924 * @returns IPRT status code.
925 * @param pVM The cross context VM structure.
926 */
927static void hmR0VmxStructsFree(PVM pVM)
928{
929 for (VMCPUID i = 0; i < pVM->cCpus; i++)
930 {
931 PVMCPU pVCpu = &pVM->aCpus[i];
932 AssertPtr(pVCpu);
933
934 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
935 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
936
937 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
938 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
939
940 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
941 }
942
943 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
944#ifdef VBOX_WITH_CRASHDUMP_MAGIC
945 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
946#endif
947}
948
949
950/**
951 * Worker function to allocate VT-x related VM structures.
952 *
953 * @returns IPRT status code.
954 * @param pVM The cross context VM structure.
955 */
956static int hmR0VmxStructsAlloc(PVM pVM)
957{
958 /*
959 * Initialize members up-front so we can cleanup properly on allocation failure.
960 */
961#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
962 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
963 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
964 pVM->hm.s.vmx.HCPhys##a_Name = 0;
965
966#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
967 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
970
971#ifdef VBOX_WITH_CRASHDUMP_MAGIC
972 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
973#endif
974 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
975
976 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
977 for (VMCPUID i = 0; i < pVM->cCpus; i++)
978 {
979 PVMCPU pVCpu = &pVM->aCpus[i];
980 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
981 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
982 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
983 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
984 }
985#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
986#undef VMXLOCAL_INIT_VM_MEMOBJ
987
988 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
989 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
990 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
991 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
992
993 /*
994 * Allocate all the VT-x structures.
995 */
996 int rc = VINF_SUCCESS;
997#ifdef VBOX_WITH_CRASHDUMP_MAGIC
998 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
999 if (RT_FAILURE(rc))
1000 goto cleanup;
1001 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1002 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1003#endif
1004
1005 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1006 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1007 {
1008 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1009 &pVM->hm.s.vmx.HCPhysApicAccess);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 /*
1015 * Initialize per-VCPU VT-x structures.
1016 */
1017 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1018 {
1019 PVMCPU pVCpu = &pVM->aCpus[i];
1020 AssertPtr(pVCpu);
1021
1022 /* Allocate the VM control structure (VMCS). */
1023 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1024 if (RT_FAILURE(rc))
1025 goto cleanup;
1026
1027 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1028 if ( PDMHasApic(pVM)
1029 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1030 {
1031 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1032 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1033 if (RT_FAILURE(rc))
1034 goto cleanup;
1035 }
1036
1037 /*
1038 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1039 * transparent accesses of specific MSRs.
1040 *
1041 * If the condition for enabling MSR bitmaps changes here, don't forget to
1042 * update HMAreMsrBitmapsAvailable().
1043 */
1044 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1045 {
1046 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1047 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1051 }
1052
1053 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1054 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1055 if (RT_FAILURE(rc))
1056 goto cleanup;
1057
1058 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1059 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1060 if (RT_FAILURE(rc))
1061 goto cleanup;
1062 }
1063
1064 return VINF_SUCCESS;
1065
1066cleanup:
1067 hmR0VmxStructsFree(pVM);
1068 return rc;
1069}
1070
1071
1072/**
1073 * Does global VT-x initialization (called during module initialization).
1074 *
1075 * @returns VBox status code.
1076 */
1077VMMR0DECL(int) VMXR0GlobalInit(void)
1078{
1079#ifdef HMVMX_USE_FUNCTION_TABLE
1080 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1081# ifdef VBOX_STRICT
1082 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1083 Assert(g_apfnVMExitHandlers[i]);
1084# endif
1085#endif
1086 return VINF_SUCCESS;
1087}
1088
1089
1090/**
1091 * Does global VT-x termination (called during module termination).
1092 */
1093VMMR0DECL(void) VMXR0GlobalTerm()
1094{
1095 /* Nothing to do currently. */
1096}
1097
1098
1099/**
1100 * Sets up and activates VT-x on the current CPU.
1101 *
1102 * @returns VBox status code.
1103 * @param pCpu Pointer to the global CPU info struct.
1104 * @param pVM The cross context VM structure. Can be
1105 * NULL after a host resume operation.
1106 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1107 * fEnabledByHost is @c true).
1108 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1109 * @a fEnabledByHost is @c true).
1110 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1111 * enable VT-x on the host.
1112 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1113 */
1114VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1115 void *pvMsrs)
1116{
1117 Assert(pCpu);
1118 Assert(pvMsrs);
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120
1121 /* Enable VT-x if it's not already enabled by the host. */
1122 if (!fEnabledByHost)
1123 {
1124 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1125 if (RT_FAILURE(rc))
1126 return rc;
1127 }
1128
1129 /*
1130 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1131 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1132 */
1133 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1134 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1135 {
1136 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1137 pCpu->fFlushAsidBeforeUse = false;
1138 }
1139 else
1140 pCpu->fFlushAsidBeforeUse = true;
1141
1142 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1143 ++pCpu->cTlbFlushes;
1144
1145 return VINF_SUCCESS;
1146}
1147
1148
1149/**
1150 * Deactivates VT-x on the current CPU.
1151 *
1152 * @returns VBox status code.
1153 * @param pCpu Pointer to the global CPU info struct.
1154 * @param pvCpuPage Pointer to the VMXON region.
1155 * @param HCPhysCpuPage Physical address of the VMXON region.
1156 *
1157 * @remarks This function should never be called when SUPR0EnableVTx() or
1158 * similar was used to enable VT-x on the host.
1159 */
1160VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1161{
1162 NOREF(pCpu);
1163 NOREF(pvCpuPage);
1164 NOREF(HCPhysCpuPage);
1165
1166 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1167 return hmR0VmxLeaveRootMode();
1168}
1169
1170
1171/**
1172 * Sets the permission bits for the specified MSR in the MSR bitmap.
1173 *
1174 * @param pVCpu The cross context virtual CPU structure.
1175 * @param uMsr The MSR value.
1176 * @param enmRead Whether reading this MSR causes a VM-exit.
1177 * @param enmWrite Whether writing this MSR causes a VM-exit.
1178 */
1179static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1180{
1181 int32_t iBit;
1182 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1183
1184 /*
1185 * Layout:
1186 * 0x000 - 0x3ff - Low MSR read bits
1187 * 0x400 - 0x7ff - High MSR read bits
1188 * 0x800 - 0xbff - Low MSR write bits
1189 * 0xc00 - 0xfff - High MSR write bits
1190 */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1194 {
1195 iBit = uMsr - UINT32_C(0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1200
1201 Assert(iBit <= 0x1fff);
1202 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1203 ASMBitSet(pbMsrBitmap, iBit);
1204 else
1205 ASMBitClear(pbMsrBitmap, iBit);
1206
1207 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1208 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1209 else
1210 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1211}
1212
1213
1214#ifdef VBOX_STRICT
1215/**
1216 * Gets the permission bits for the specified MSR in the MSR bitmap.
1217 *
1218 * @returns VBox status code.
1219 * @retval VINF_SUCCESS if the specified MSR is found.
1220 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1221 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1222 *
1223 * @param pVCpu The cross context virtual CPU structure.
1224 * @param uMsr The MSR.
1225 * @param penmRead Where to store the read permissions.
1226 * @param penmWrite Where to store the write permissions.
1227 */
1228static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1229{
1230 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1231 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1232 int32_t iBit;
1233 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1234
1235 /* See hmR0VmxSetMsrPermission() for the layout. */
1236 if (uMsr <= 0x00001FFF)
1237 iBit = uMsr;
1238 else if ( uMsr >= 0xC0000000
1239 && uMsr <= 0xC0001FFF)
1240 {
1241 iBit = (uMsr - 0xC0000000);
1242 pbMsrBitmap += 0x400;
1243 }
1244 else
1245 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1246
1247 Assert(iBit <= 0x1fff);
1248 if (ASMBitTest(pbMsrBitmap, iBit))
1249 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1250 else
1251 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1252
1253 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1254 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1255 else
1256 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1257 return VINF_SUCCESS;
1258}
1259#endif /* VBOX_STRICT */
1260
1261
1262/**
1263 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1264 * area.
1265 *
1266 * @returns VBox status code.
1267 * @param pVCpu The cross context virtual CPU structure.
1268 * @param cMsrs The number of MSRs.
1269 */
1270DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1271{
1272 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1273 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1274 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1275 {
1276 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1277 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1278 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1279 }
1280
1281 /* Update number of guest MSRs to load/store across the world-switch. */
1282 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1283 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1284
1285 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1286 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1287 AssertRCReturn(rc, rc);
1288
1289 /* Update the VCPU's copy of the MSR count. */
1290 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1291
1292 return VINF_SUCCESS;
1293}
1294
1295
1296/**
1297 * Adds a new (or updates the value of an existing) guest/host MSR
1298 * pair to be swapped during the world-switch as part of the
1299 * auto-load/store MSR area in the VMCS.
1300 *
1301 * @returns VBox status code.
1302 * @param pVCpu The cross context virtual CPU structure.
1303 * @param uMsr The MSR.
1304 * @param uGuestMsrValue Value of the guest MSR.
1305 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1306 * necessary.
1307 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1308 * its value was updated. Optional, can be NULL.
1309 */
1310static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1311 bool *pfAddedAndUpdated)
1312{
1313 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315 uint32_t i;
1316 for (i = 0; i < cMsrs; i++)
1317 {
1318 if (pGuestMsr->u32Msr == uMsr)
1319 break;
1320 pGuestMsr++;
1321 }
1322
1323 bool fAdded = false;
1324 if (i == cMsrs)
1325 {
1326 ++cMsrs;
1327 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1328 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1329
1330 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1331 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1332 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1333
1334 fAdded = true;
1335 }
1336
1337 /* Update the MSR values in the auto-load/store MSR area. */
1338 pGuestMsr->u32Msr = uMsr;
1339 pGuestMsr->u64Value = uGuestMsrValue;
1340
1341 /* Create/update the MSR slot in the host MSR area. */
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 pHostMsr += i;
1344 pHostMsr->u32Msr = uMsr;
1345
1346 /*
1347 * Update the host MSR only when requested by the caller AND when we're
1348 * adding it to the auto-load/store area. Otherwise, it would have been
1349 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1350 */
1351 bool fUpdatedMsrValue = false;
1352 if ( fAdded
1353 && fUpdateHostMsr)
1354 {
1355 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1356 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1357 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1358 fUpdatedMsrValue = true;
1359 }
1360
1361 if (pfAddedAndUpdated)
1362 *pfAddedAndUpdated = fUpdatedMsrValue;
1363 return VINF_SUCCESS;
1364}
1365
1366
1367/**
1368 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1369 * auto-load/store MSR area in the VMCS.
1370 *
1371 * @returns VBox status code.
1372 * @param pVCpu The cross context virtual CPU structure.
1373 * @param uMsr The MSR.
1374 */
1375static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1376{
1377 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1379 for (uint32_t i = 0; i < cMsrs; i++)
1380 {
1381 /* Find the MSR. */
1382 if (pGuestMsr->u32Msr == uMsr)
1383 {
1384 /* If it's the last MSR, simply reduce the count. */
1385 if (i == cMsrs - 1)
1386 {
1387 --cMsrs;
1388 break;
1389 }
1390
1391 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1392 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1393 pLastGuestMsr += cMsrs - 1;
1394 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1395 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1396
1397 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1398 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1399 pLastHostMsr += cMsrs - 1;
1400 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1401 pHostMsr->u64Value = pLastHostMsr->u64Value;
1402 --cMsrs;
1403 break;
1404 }
1405 pGuestMsr++;
1406 }
1407
1408 /* Update the VMCS if the count changed (meaning the MSR was found). */
1409 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1410 {
1411 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1412 AssertRCReturn(rc, rc);
1413
1414 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1415 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1416 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1417
1418 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1419 return VINF_SUCCESS;
1420 }
1421
1422 return VERR_NOT_FOUND;
1423}
1424
1425
1426/**
1427 * Checks if the specified guest MSR is part of the auto-load/store area in
1428 * the VMCS.
1429 *
1430 * @returns true if found, false otherwise.
1431 * @param pVCpu The cross context virtual CPU structure.
1432 * @param uMsr The MSR to find.
1433 */
1434static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1435{
1436 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1437 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1438
1439 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1440 {
1441 if (pGuestMsr->u32Msr == uMsr)
1442 return true;
1443 }
1444 return false;
1445}
1446
1447
1448/**
1449 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1450 *
1451 * @param pVCpu The cross context virtual CPU structure.
1452 *
1453 * @remarks No-long-jump zone!!!
1454 */
1455static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1456{
1457 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1458 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1459 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1460 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1461
1462 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1463 {
1464 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1465
1466 /*
1467 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1468 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1469 */
1470 if (pHostMsr->u32Msr == MSR_K6_EFER)
1471 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1472 else
1473 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1474 }
1475
1476 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1477}
1478
1479
1480/**
1481 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1482 * perform lazy restoration of the host MSRs while leaving VT-x.
1483 *
1484 * @param pVCpu The cross context virtual CPU structure.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491
1492 /*
1493 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1494 */
1495 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1496 {
1497 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1498#if HC_ARCH_BITS == 64
1499 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1500 {
1501 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1502 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1503 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1504 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1505 }
1506#endif
1507 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1508 }
1509}
1510
1511
1512/**
1513 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1514 * lazily while leaving VT-x.
1515 *
1516 * @returns true if it does, false otherwise.
1517 * @param pVCpu The cross context virtual CPU structure.
1518 * @param uMsr The MSR to check.
1519 */
1520static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1521{
1522 NOREF(pVCpu);
1523#if HC_ARCH_BITS == 64
1524 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1525 {
1526 switch (uMsr)
1527 {
1528 case MSR_K8_LSTAR:
1529 case MSR_K6_STAR:
1530 case MSR_K8_SF_MASK:
1531 case MSR_K8_KERNEL_GS_BASE:
1532 return true;
1533 }
1534 }
1535#else
1536 RT_NOREF(pVCpu, uMsr);
1537#endif
1538 return false;
1539}
1540
1541
1542/**
1543 * Saves a set of guest MSRs back into the guest-CPU context.
1544 *
1545 * @param pVCpu The cross context virtual CPU structure.
1546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1547 * out-of-sync. Make sure to update the required fields
1548 * before using them.
1549 *
1550 * @remarks No-long-jump zone!!!
1551 */
1552static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1553{
1554 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1555 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1556
1557 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1558 {
1559 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1560#if HC_ARCH_BITS == 64
1561 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1562 {
1563 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1564 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1565 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1566 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1567 }
1568#else
1569 NOREF(pMixedCtx);
1570#endif
1571 }
1572}
1573
1574
1575/**
1576 * Loads a set of guests MSRs to allow read/passthru to the guest.
1577 *
1578 * The name of this function is slightly confusing. This function does NOT
1579 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1580 * common prefix for functions dealing with "lazy restoration" of the shared
1581 * MSRs.
1582 *
1583 * @param pVCpu The cross context virtual CPU structure.
1584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1585 * out-of-sync. Make sure to update the required fields
1586 * before using them.
1587 *
1588 * @remarks No-long-jump zone!!!
1589 */
1590static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1596#if HC_ARCH_BITS == 64
1597 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1598 {
1599 /*
1600 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1601 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1602 * we can skip a few MSR writes.
1603 *
1604 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1605 * guest MSR values in the guest-CPU context might be different to what's currently
1606 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1607 * CPU, see @bugref{8728}.
1608 */
1609 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1610 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1611 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1612 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1613 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1614 {
1615#ifdef VBOX_STRICT
1616 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1617 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1618 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1619 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1620#endif
1621 }
1622 else
1623 {
1624 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1625 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1626 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1627 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1628 }
1629 }
1630#else
1631 RT_NOREF(pMixedCtx);
1632#endif
1633 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1634}
1635
1636
1637/**
1638 * Performs lazy restoration of the set of host MSRs if they were previously
1639 * loaded with guest MSR values.
1640 *
1641 * @param pVCpu The cross context virtual CPU structure.
1642 *
1643 * @remarks No-long-jump zone!!!
1644 * @remarks The guest MSRs should have been saved back into the guest-CPU
1645 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1646 */
1647static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1648{
1649 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1650 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1651
1652 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1653 {
1654 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1655#if HC_ARCH_BITS == 64
1656 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1657 {
1658 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1659 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1660 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1661 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1662 }
1663#endif
1664 }
1665 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1666}
1667
1668
1669/**
1670 * Verifies that our cached values of the VMCS controls are all
1671 * consistent with what's actually present in the VMCS.
1672 *
1673 * @returns VBox status code.
1674 * @param pVCpu The cross context virtual CPU structure.
1675 */
1676static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1677{
1678 uint32_t u32Val;
1679 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1680 AssertRCReturn(rc, rc);
1681 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1682 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1683
1684 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1685 AssertRCReturn(rc, rc);
1686 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1687 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1688
1689 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1690 AssertRCReturn(rc, rc);
1691 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1692 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1695 AssertRCReturn(rc, rc);
1696 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1697 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1698
1699 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1700 {
1701 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1702 AssertRCReturn(rc, rc);
1703 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1704 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1705 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1706 }
1707
1708 return VINF_SUCCESS;
1709}
1710
1711
1712#ifdef VBOX_STRICT
1713/**
1714 * Verifies that our cached host EFER value has not changed
1715 * since we cached it.
1716 *
1717 * @param pVCpu The cross context virtual CPU structure.
1718 */
1719static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1720{
1721 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1722
1723 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1724 {
1725 uint64_t u64Val;
1726 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1727 AssertRC(rc);
1728
1729 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1730 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1731 }
1732}
1733
1734
1735/**
1736 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1737 * VMCS are correct.
1738 *
1739 * @param pVCpu The cross context virtual CPU structure.
1740 */
1741static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1742{
1743 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1744
1745 /* Verify MSR counts in the VMCS are what we think it should be. */
1746 uint32_t cMsrs;
1747 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1748 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1749
1750 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1751 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1752
1753 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1754 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1755
1756 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1757 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1758 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1759 {
1760 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1761 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1762 pGuestMsr->u32Msr, cMsrs));
1763
1764 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1765 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1766 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1767
1768 /* Verify that the permissions are as expected in the MSR bitmap. */
1769 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1770 {
1771 VMXMSREXITREAD enmRead;
1772 VMXMSREXITWRITE enmWrite;
1773 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1774 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1775 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1776 {
1777 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1778 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1779 }
1780 else
1781 {
1782 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1783 pGuestMsr->u32Msr, cMsrs));
1784 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1785 pGuestMsr->u32Msr, cMsrs));
1786 }
1787 }
1788 }
1789}
1790#endif /* VBOX_STRICT */
1791
1792
1793/**
1794 * Flushes the TLB using EPT.
1795 *
1796 * @returns VBox status code.
1797 * @param pVCpu The cross context virtual CPU structure of the calling
1798 * EMT. Can be NULL depending on @a enmFlush.
1799 * @param enmFlush Type of flush.
1800 *
1801 * @remarks Caller is responsible for making sure this function is called only
1802 * when NestedPaging is supported and providing @a enmFlush that is
1803 * supported by the CPU.
1804 * @remarks Can be called with interrupts disabled.
1805 */
1806static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1807{
1808 uint64_t au64Descriptor[2];
1809 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1810 au64Descriptor[0] = 0;
1811 else
1812 {
1813 Assert(pVCpu);
1814 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1815 }
1816 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1817
1818 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1819 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1820 rc));
1821 if ( RT_SUCCESS(rc)
1822 && pVCpu)
1823 {
1824 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1825 }
1826}
1827
1828
1829/**
1830 * Flushes the TLB using VPID.
1831 *
1832 * @returns VBox status code.
1833 * @param pVM The cross context VM structure.
1834 * @param pVCpu The cross context virtual CPU structure of the calling
1835 * EMT. Can be NULL depending on @a enmFlush.
1836 * @param enmFlush Type of flush.
1837 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1838 * on @a enmFlush).
1839 *
1840 * @remarks Can be called with interrupts disabled.
1841 */
1842static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1843{
1844 NOREF(pVM);
1845 AssertPtr(pVM);
1846 Assert(pVM->hm.s.vmx.fVpid);
1847
1848 uint64_t au64Descriptor[2];
1849 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1850 {
1851 au64Descriptor[0] = 0;
1852 au64Descriptor[1] = 0;
1853 }
1854 else
1855 {
1856 AssertPtr(pVCpu);
1857 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1858 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1859 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1860 au64Descriptor[1] = GCPtr;
1861 }
1862
1863 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1864 AssertMsg(rc == VINF_SUCCESS,
1865 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1866 if ( RT_SUCCESS(rc)
1867 && pVCpu)
1868 {
1869 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1870 }
1871}
1872
1873
1874/**
1875 * Invalidates a guest page by guest virtual address. Only relevant for
1876 * EPT/VPID, otherwise there is nothing really to invalidate.
1877 *
1878 * @returns VBox status code.
1879 * @param pVM The cross context VM structure.
1880 * @param pVCpu The cross context virtual CPU structure.
1881 * @param GCVirt Guest virtual address of the page to invalidate.
1882 */
1883VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1884{
1885 AssertPtr(pVM);
1886 AssertPtr(pVCpu);
1887 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1888
1889 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1890 if (!fFlushPending)
1891 {
1892 /*
1893 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1894 * See @bugref{6043} and @bugref{6177}.
1895 *
1896 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1897 * function maybe called in a loop with individual addresses.
1898 */
1899 if (pVM->hm.s.vmx.fVpid)
1900 {
1901 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1902
1903#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1904 /*
1905 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1906 * where executing INVVPID outside 64-bit mode does not flush translations of
1907 * 64-bit linear addresses, see @bugref{6208#c72}.
1908 */
1909 if (RT_HI_U32(GCVirt))
1910 fVpidFlush = false;
1911#endif
1912
1913 if (fVpidFlush)
1914 {
1915 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1916 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1917 }
1918 else
1919 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1920 }
1921 else if (pVM->hm.s.fNestedPaging)
1922 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1923 }
1924
1925 return VINF_SUCCESS;
1926}
1927
1928
1929/**
1930 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1931 * otherwise there is nothing really to invalidate.
1932 *
1933 * @returns VBox status code.
1934 * @param pVM The cross context VM structure.
1935 * @param pVCpu The cross context virtual CPU structure.
1936 * @param GCPhys Guest physical address of the page to invalidate.
1937 */
1938VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1939{
1940 NOREF(pVM); NOREF(GCPhys);
1941 LogFlowFunc(("%RGp\n", GCPhys));
1942
1943 /*
1944 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1945 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1946 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1947 */
1948 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1949 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1950 return VINF_SUCCESS;
1951}
1952
1953
1954/**
1955 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1956 * case where neither EPT nor VPID is supported by the CPU.
1957 *
1958 * @param pVM The cross context VM structure.
1959 * @param pVCpu The cross context virtual CPU structure.
1960 * @param pCpu Pointer to the global HM struct.
1961 *
1962 * @remarks Called with interrupts disabled.
1963 */
1964static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1965{
1966 AssertPtr(pVCpu);
1967 AssertPtr(pCpu);
1968 NOREF(pVM);
1969
1970 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1971
1972 Assert(pCpu->idCpu != NIL_RTCPUID);
1973 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1974 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1975 pVCpu->hm.s.fForceTLBFlush = false;
1976 return;
1977}
1978
1979
1980/**
1981 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1982 *
1983 * @param pVM The cross context VM structure.
1984 * @param pVCpu The cross context virtual CPU structure.
1985 * @param pCpu Pointer to the global HM CPU struct.
1986 * @remarks All references to "ASID" in this function pertains to "VPID" in
1987 * Intel's nomenclature. The reason is, to avoid confusion in compare
1988 * statements since the host-CPU copies are named "ASID".
1989 *
1990 * @remarks Called with interrupts disabled.
1991 */
1992static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1993{
1994#ifdef VBOX_WITH_STATISTICS
1995 bool fTlbFlushed = false;
1996# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1997# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1998 if (!fTlbFlushed) \
1999 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2000 } while (0)
2001#else
2002# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2003# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2004#endif
2005
2006 AssertPtr(pVM);
2007 AssertPtr(pCpu);
2008 AssertPtr(pVCpu);
2009 Assert(pCpu->idCpu != NIL_RTCPUID);
2010
2011 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2012 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2013 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2014
2015 /*
2016 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2017 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2018 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2019 */
2020 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2021 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2022 {
2023 ++pCpu->uCurrentAsid;
2024 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2025 {
2026 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2027 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2028 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2029 }
2030
2031 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2032 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2033 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2034
2035 /*
2036 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2037 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2038 */
2039 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041 HMVMX_SET_TAGGED_TLB_FLUSHED();
2042 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 /*
2049 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2050 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2051 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2052 * but not guest-physical mappings.
2053 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2054 */
2055 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2057 HMVMX_SET_TAGGED_TLB_FLUSHED();
2058 }
2059
2060 pVCpu->hm.s.fForceTLBFlush = false;
2061 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2062
2063 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2064 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2065 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2066 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2067 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2068 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2069 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2070 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073 /* Update VMCS with the VPID. */
2074 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2075 AssertRC(rc);
2076
2077#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2078}
2079
2080
2081/**
2082 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2083 *
2084 * @returns VBox status code.
2085 * @param pVM The cross context VM structure.
2086 * @param pVCpu The cross context virtual CPU structure.
2087 * @param pCpu Pointer to the global HM CPU struct.
2088 *
2089 * @remarks Called with interrupts disabled.
2090 */
2091static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2092{
2093 AssertPtr(pVM);
2094 AssertPtr(pVCpu);
2095 AssertPtr(pCpu);
2096 Assert(pCpu->idCpu != NIL_RTCPUID);
2097 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2098 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2099
2100 /*
2101 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2102 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2103 */
2104 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2105 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2109 }
2110
2111 /* Check for explicit TLB flushes. */
2112 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2113 {
2114 pVCpu->hm.s.fForceTLBFlush = true;
2115 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2116 }
2117
2118 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120
2121 if (pVCpu->hm.s.fForceTLBFlush)
2122 {
2123 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 }
2126}
2127
2128
2129/**
2130 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2131 *
2132 * @returns VBox status code.
2133 * @param pVM The cross context VM structure.
2134 * @param pVCpu The cross context virtual CPU structure.
2135 * @param pCpu Pointer to the global HM CPU struct.
2136 *
2137 * @remarks Called with interrupts disabled.
2138 */
2139static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2140{
2141 AssertPtr(pVM);
2142 AssertPtr(pVCpu);
2143 AssertPtr(pCpu);
2144 Assert(pCpu->idCpu != NIL_RTCPUID);
2145 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2146 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2147
2148 /*
2149 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2150 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2151 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2152 */
2153 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2154 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2155 {
2156 pVCpu->hm.s.fForceTLBFlush = true;
2157 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2158 }
2159
2160 /* Check for explicit TLB flushes. */
2161 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2162 {
2163 /*
2164 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2165 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2166 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2167 */
2168 pVCpu->hm.s.fForceTLBFlush = true;
2169 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2170 }
2171
2172 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2173 if (pVCpu->hm.s.fForceTLBFlush)
2174 {
2175 ++pCpu->uCurrentAsid;
2176 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2177 {
2178 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2179 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2180 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2181 }
2182
2183 pVCpu->hm.s.fForceTLBFlush = false;
2184 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2185 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2186 if (pCpu->fFlushAsidBeforeUse)
2187 {
2188 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2189 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2190 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2191 {
2192 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2193 pCpu->fFlushAsidBeforeUse = false;
2194 }
2195 else
2196 {
2197 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2198 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2199 }
2200 }
2201 }
2202
2203 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2204 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2205 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2206 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2207 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2208 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2209 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2210
2211 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2212 AssertRC(rc);
2213}
2214
2215
2216/**
2217 * Flushes the guest TLB entry based on CPU capabilities.
2218 *
2219 * @param pVCpu The cross context virtual CPU structure.
2220 * @param pCpu Pointer to the global HM CPU struct.
2221 */
2222DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2223{
2224#ifdef HMVMX_ALWAYS_FLUSH_TLB
2225 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2226#endif
2227 PVM pVM = pVCpu->CTX_SUFF(pVM);
2228 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2229 {
2230 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2231 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2232 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2233 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2234 default:
2235 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2236 break;
2237 }
2238
2239 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2240}
2241
2242
2243/**
2244 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2245 * TLB entries from the host TLB before VM-entry.
2246 *
2247 * @returns VBox status code.
2248 * @param pVM The cross context VM structure.
2249 */
2250static int hmR0VmxSetupTaggedTlb(PVM pVM)
2251{
2252 /*
2253 * Determine optimal flush type for Nested Paging.
2254 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2255 * guest execution (see hmR3InitFinalizeR0()).
2256 */
2257 if (pVM->hm.s.fNestedPaging)
2258 {
2259 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2260 {
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2262 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2263 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2264 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2265 else
2266 {
2267 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2268 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2269 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2271 }
2272
2273 /* Make sure the write-back cacheable memory type for EPT is supported. */
2274 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2275 {
2276 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2277 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2278 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2279 }
2280
2281 /* EPT requires a page-walk length of 4. */
2282 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2283 {
2284 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2285 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2286 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2287 }
2288 }
2289 else
2290 {
2291 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2292 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2293 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2294 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2295 }
2296 }
2297
2298 /*
2299 * Determine optimal flush type for VPID.
2300 */
2301 if (pVM->hm.s.vmx.fVpid)
2302 {
2303 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2304 {
2305 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2306 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2307 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2308 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2309 else
2310 {
2311 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2312 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2313 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2314 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2315 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2316 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2317 pVM->hm.s.vmx.fVpid = false;
2318 }
2319 }
2320 else
2321 {
2322 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2323 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2324 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2325 pVM->hm.s.vmx.fVpid = false;
2326 }
2327 }
2328
2329 /*
2330 * Setup the handler for flushing tagged-TLBs.
2331 */
2332 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2333 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2334 else if (pVM->hm.s.fNestedPaging)
2335 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2336 else if (pVM->hm.s.vmx.fVpid)
2337 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2338 else
2339 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2340 return VINF_SUCCESS;
2341}
2342
2343
2344/**
2345 * Sets up pin-based VM-execution controls in the VMCS.
2346 *
2347 * @returns VBox status code.
2348 * @param pVM The cross context VM structure.
2349 * @param pVCpu The cross context virtual CPU structure.
2350 */
2351static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2352{
2353 AssertPtr(pVM);
2354 AssertPtr(pVCpu);
2355
2356 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2357 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2358
2359 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2360 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2361
2362 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2363 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2364
2365 /* Enable the VMX preemption timer. */
2366 if (pVM->hm.s.vmx.fUsePreemptTimer)
2367 {
2368 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2369 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2370 }
2371
2372#if 0
2373 /* Enable posted-interrupt processing. */
2374 if (pVM->hm.s.fPostedIntrs)
2375 {
2376 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2377 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2378 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2379 }
2380#endif
2381
2382 if ((val & zap) != val)
2383 {
2384 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2385 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2386 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2387 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2388 }
2389
2390 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2391 AssertRCReturn(rc, rc);
2392
2393 pVCpu->hm.s.vmx.u32PinCtls = val;
2394 return rc;
2395}
2396
2397
2398/**
2399 * Sets up processor-based VM-execution controls in the VMCS.
2400 *
2401 * @returns VBox status code.
2402 * @param pVM The cross context VM structure.
2403 * @param pVCpu The cross context virtual CPU structure.
2404 */
2405static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2406{
2407 AssertPtr(pVM);
2408 AssertPtr(pVCpu);
2409
2410 int rc = VERR_INTERNAL_ERROR_5;
2411 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2412 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2413
2414 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2415 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2416 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2417 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2419 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2421
2422 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2423 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2424 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2425 {
2426 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2427 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2428 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2429 }
2430
2431 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2432 if (!pVM->hm.s.fNestedPaging)
2433 {
2434 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2435 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2436 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2437 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2438 }
2439
2440 /* Use TPR shadowing if supported by the CPU. */
2441 if ( PDMHasApic(pVM)
2442 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2443 {
2444 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2445 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2446 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2447 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2448 AssertRCReturn(rc, rc);
2449
2450 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2451 /* CR8 writes cause a VM-exit based on TPR threshold. */
2452 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2453 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2454 }
2455 else
2456 {
2457 /*
2458 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2459 * Set this control only for 64-bit guests.
2460 */
2461 if (pVM->hm.s.fAllow64BitGuests)
2462 {
2463 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2464 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2465 }
2466 }
2467
2468 /* Use MSR-bitmaps if supported by the CPU. */
2469 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2470 {
2471 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2472
2473 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2474 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2475 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2476 AssertRCReturn(rc, rc);
2477
2478 /*
2479 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2480 * automatically using dedicated fields in the VMCS.
2481 */
2482 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2483 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2484 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2485 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2486 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2487
2488#if HC_ARCH_BITS == 64
2489 /*
2490 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2491 */
2492 if (pVM->hm.s.fAllow64BitGuests)
2493 {
2494 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2495 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2498 }
2499#endif
2500 /*
2501 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2502 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2503 */
2504 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506
2507 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2508 }
2509
2510 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2511 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2512 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2513
2514 if ((val & zap) != val)
2515 {
2516 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2517 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2518 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2519 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2520 }
2521
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2523 AssertRCReturn(rc, rc);
2524
2525 pVCpu->hm.s.vmx.u32ProcCtls = val;
2526
2527 /*
2528 * Secondary processor-based VM-execution controls.
2529 */
2530 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2531 {
2532 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2533 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2534
2535 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2537
2538 if (pVM->hm.s.fNestedPaging)
2539 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2540
2541 /*
2542 * Enable the INVPCID instruction if supported by the hardware and we expose
2543 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2544 */
2545 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2546 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2547 {
2548 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2549 }
2550
2551 if (pVM->hm.s.vmx.fVpid)
2552 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2553
2554 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2555 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2556
2557#if 0
2558 if (pVM->hm.s.fVirtApicRegs)
2559 {
2560 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2561 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2562
2563 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2564 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2565 }
2566#endif
2567
2568 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2569 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2570 * done dynamically. */
2571 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2572 {
2573 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2574 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2575 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2576 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2577 AssertRCReturn(rc, rc);
2578 }
2579
2580 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2581 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2582
2583 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2584 && pVM->hm.s.vmx.cPleGapTicks
2585 && pVM->hm.s.vmx.cPleWindowTicks)
2586 {
2587 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2588
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2591 AssertRCReturn(rc, rc);
2592 }
2593
2594 if ((val & zap) != val)
2595 {
2596 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2597 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2598 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2599 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2600 }
2601
2602 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2603 AssertRCReturn(rc, rc);
2604
2605 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2606 }
2607 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2608 {
2609 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2610 "available\n"));
2611 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2612 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2613 }
2614
2615 return VINF_SUCCESS;
2616}
2617
2618
2619/**
2620 * Sets up miscellaneous (everything other than Pin & Processor-based
2621 * VM-execution) control fields in the VMCS.
2622 *
2623 * @returns VBox status code.
2624 * @param pVM The cross context VM structure.
2625 * @param pVCpu The cross context virtual CPU structure.
2626 */
2627static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2628{
2629 NOREF(pVM);
2630 AssertPtr(pVM);
2631 AssertPtr(pVCpu);
2632
2633 int rc = VERR_GENERAL_FAILURE;
2634
2635 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2636#if 0
2637 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2639 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2640
2641 /*
2642 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2643 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2644 * We thus use the exception bitmap to control it rather than use both.
2645 */
2646 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2647 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2648
2649 /** @todo Explore possibility of using IO-bitmaps. */
2650 /* All IO & IOIO instructions cause VM-exits. */
2651 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2652 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2653
2654 /* Initialize the MSR-bitmap area. */
2655 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2656 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2657 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2658 AssertRCReturn(rc, rc);
2659#endif
2660
2661 /* Setup MSR auto-load/store area. */
2662 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2663 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2664 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2665 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2666 AssertRCReturn(rc, rc);
2667
2668 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2669 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2670 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2671 AssertRCReturn(rc, rc);
2672
2673 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2674 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2675 AssertRCReturn(rc, rc);
2676
2677 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2678#if 0
2679 /* Setup debug controls */
2680 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2681 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2682 AssertRCReturn(rc, rc);
2683#endif
2684
2685 return rc;
2686}
2687
2688
2689/**
2690 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2691 *
2692 * We shall setup those exception intercepts that don't change during the
2693 * lifetime of the VM here. The rest are done dynamically while loading the
2694 * guest state.
2695 *
2696 * @returns VBox status code.
2697 * @param pVM The cross context VM structure.
2698 * @param pVCpu The cross context virtual CPU structure.
2699 */
2700static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2701{
2702 AssertPtr(pVM);
2703 AssertPtr(pVCpu);
2704
2705 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2706
2707 uint32_t u32XcptBitmap = 0;
2708
2709 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2710 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2711
2712 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2713 and writes, and because recursive #DBs can cause the CPU hang, we must always
2714 intercept #DB. */
2715 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2716
2717 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2718 if (!pVM->hm.s.fNestedPaging)
2719 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2720
2721 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2722 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2723 AssertRCReturn(rc, rc);
2724 return rc;
2725}
2726
2727
2728/**
2729 * Sets up the initial guest-state mask. The guest-state mask is consulted
2730 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2731 * for the nested virtualization case (as it would cause a VM-exit).
2732 *
2733 * @param pVCpu The cross context virtual CPU structure.
2734 */
2735static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2736{
2737 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2738 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2739 return VINF_SUCCESS;
2740}
2741
2742
2743/**
2744 * Does per-VM VT-x initialization.
2745 *
2746 * @returns VBox status code.
2747 * @param pVM The cross context VM structure.
2748 */
2749VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2750{
2751 LogFlowFunc(("pVM=%p\n", pVM));
2752
2753 int rc = hmR0VmxStructsAlloc(pVM);
2754 if (RT_FAILURE(rc))
2755 {
2756 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2757 return rc;
2758 }
2759
2760 return VINF_SUCCESS;
2761}
2762
2763
2764/**
2765 * Does per-VM VT-x termination.
2766 *
2767 * @returns VBox status code.
2768 * @param pVM The cross context VM structure.
2769 */
2770VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2771{
2772 LogFlowFunc(("pVM=%p\n", pVM));
2773
2774#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2775 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2776 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2777#endif
2778 hmR0VmxStructsFree(pVM);
2779 return VINF_SUCCESS;
2780}
2781
2782
2783/**
2784 * Sets up the VM for execution under VT-x.
2785 * This function is only called once per-VM during initialization.
2786 *
2787 * @returns VBox status code.
2788 * @param pVM The cross context VM structure.
2789 */
2790VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2791{
2792 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2794
2795 LogFlowFunc(("pVM=%p\n", pVM));
2796
2797 /*
2798 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2799 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2800 */
2801 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2802 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2803 || !pVM->hm.s.vmx.pRealModeTSS))
2804 {
2805 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2806 return VERR_INTERNAL_ERROR;
2807 }
2808
2809 /* Initialize these always, see hmR3InitFinalizeR0().*/
2810 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2811 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2812
2813 /* Setup the tagged-TLB flush handlers. */
2814 int rc = hmR0VmxSetupTaggedTlb(pVM);
2815 if (RT_FAILURE(rc))
2816 {
2817 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2818 return rc;
2819 }
2820
2821 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2822 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2823#if HC_ARCH_BITS == 64
2824 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2825 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2826 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2827 {
2828 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2829 }
2830#endif
2831
2832 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2833 RTCCUINTREG uHostCR4 = ASMGetCR4();
2834 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2835 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2836
2837 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2838 {
2839 PVMCPU pVCpu = &pVM->aCpus[i];
2840 AssertPtr(pVCpu);
2841 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2842
2843 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2844 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2845
2846 /* Set revision dword at the beginning of the VMCS structure. */
2847 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2848
2849 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2850 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2851 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2852 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2853
2854 /* Load this VMCS as the current VMCS. */
2855 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2856 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2857 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2858
2859 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2860 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2861 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2862
2863 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2864 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2865 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2866
2867 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2868 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2869 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2870
2871 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2872 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2873 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2874
2875 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2876 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2877 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2878
2879#if HC_ARCH_BITS == 32
2880 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2881 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2882 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2883#endif
2884
2885 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2886 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2887 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2888 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2889
2890 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2891
2892 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2893 }
2894
2895 return VINF_SUCCESS;
2896}
2897
2898
2899/**
2900 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2901 * the VMCS.
2902 *
2903 * @returns VBox status code.
2904 * @param pVM The cross context VM structure.
2905 * @param pVCpu The cross context virtual CPU structure.
2906 */
2907DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2908{
2909 NOREF(pVM); NOREF(pVCpu);
2910
2911 RTCCUINTREG uReg = ASMGetCR0();
2912 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2913 AssertRCReturn(rc, rc);
2914
2915 uReg = ASMGetCR3();
2916 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2917 AssertRCReturn(rc, rc);
2918
2919 uReg = ASMGetCR4();
2920 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2921 AssertRCReturn(rc, rc);
2922 return rc;
2923}
2924
2925
2926#if HC_ARCH_BITS == 64
2927/**
2928 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2929 * requirements. See hmR0VmxSaveHostSegmentRegs().
2930 */
2931# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2932 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2933 { \
2934 bool fValidSelector = true; \
2935 if ((selValue) & X86_SEL_LDT) \
2936 { \
2937 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2938 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2939 } \
2940 if (fValidSelector) \
2941 { \
2942 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2943 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2944 } \
2945 (selValue) = 0; \
2946 }
2947#endif
2948
2949
2950/**
2951 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2952 * the host-state area in the VMCS.
2953 *
2954 * @returns VBox status code.
2955 * @param pVM The cross context VM structure.
2956 * @param pVCpu The cross context virtual CPU structure.
2957 */
2958DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2959{
2960 int rc = VERR_INTERNAL_ERROR_5;
2961
2962#if HC_ARCH_BITS == 64
2963 /*
2964 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2965 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2966 *
2967 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2968 * Was observed booting Solaris10u10 32-bit guest.
2969 */
2970 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2971 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2972 {
2973 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2974 pVCpu->idCpu));
2975 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2976 }
2977 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2978#else
2979 RT_NOREF(pVCpu);
2980#endif
2981
2982 /*
2983 * Host DS, ES, FS and GS segment registers.
2984 */
2985#if HC_ARCH_BITS == 64
2986 RTSEL uSelDS = ASMGetDS();
2987 RTSEL uSelES = ASMGetES();
2988 RTSEL uSelFS = ASMGetFS();
2989 RTSEL uSelGS = ASMGetGS();
2990#else
2991 RTSEL uSelDS = 0;
2992 RTSEL uSelES = 0;
2993 RTSEL uSelFS = 0;
2994 RTSEL uSelGS = 0;
2995#endif
2996
2997 /*
2998 * Host CS and SS segment registers.
2999 */
3000 RTSEL uSelCS = ASMGetCS();
3001 RTSEL uSelSS = ASMGetSS();
3002
3003 /*
3004 * Host TR segment register.
3005 */
3006 RTSEL uSelTR = ASMGetTR();
3007
3008#if HC_ARCH_BITS == 64
3009 /*
3010 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3011 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3012 */
3013 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3014 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3015 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3016 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3017# undef VMXLOCAL_ADJUST_HOST_SEG
3018#endif
3019
3020 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3021 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3022 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3023 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3024 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3025 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3026 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3027 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3028 Assert(uSelCS);
3029 Assert(uSelTR);
3030
3031 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3032#if 0
3033 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3034 Assert(uSelSS != 0);
3035#endif
3036
3037 /* Write these host selector fields into the host-state area in the VMCS. */
3038 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3039 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3040#if HC_ARCH_BITS == 64
3041 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3042 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3043 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3044 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3045#else
3046 NOREF(uSelDS);
3047 NOREF(uSelES);
3048 NOREF(uSelFS);
3049 NOREF(uSelGS);
3050#endif
3051 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3052 AssertRCReturn(rc, rc);
3053
3054 /*
3055 * Host GDTR and IDTR.
3056 */
3057 RTGDTR Gdtr;
3058 RTIDTR Idtr;
3059 RT_ZERO(Gdtr);
3060 RT_ZERO(Idtr);
3061 ASMGetGDTR(&Gdtr);
3062 ASMGetIDTR(&Idtr);
3063 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3064 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3065 AssertRCReturn(rc, rc);
3066
3067#if HC_ARCH_BITS == 64
3068 /*
3069 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3070 * maximum limit (0xffff) on every VM-exit.
3071 */
3072 if (Gdtr.cbGdt != 0xffff)
3073 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3074
3075 /*
3076 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3077 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3078 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3079 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3080 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3081 * hosts where we are pretty sure it won't cause trouble.
3082 */
3083# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3084 if (Idtr.cbIdt < 0x0fff)
3085# else
3086 if (Idtr.cbIdt != 0xffff)
3087# endif
3088 {
3089 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3090 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3091 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3092 }
3093#endif
3094
3095 /*
3096 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3097 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3098 */
3099 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3100 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3101 VERR_VMX_INVALID_HOST_STATE);
3102
3103 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3104#if HC_ARCH_BITS == 64
3105 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3106
3107 /*
3108 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3109 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3110 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3111 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3112 *
3113 * [1] See Intel spec. 3.5 "System Descriptor Types".
3114 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3115 */
3116 Assert(pDesc->System.u4Type == 11);
3117 if ( pDesc->System.u16LimitLow != 0x67
3118 || pDesc->System.u4LimitHigh)
3119 {
3120 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3121 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3122 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3123 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3124 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3125 }
3126
3127 /*
3128 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3129 */
3130 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3131 {
3132 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3133 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3134 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3135 {
3136 /* The GDT is read-only but the writable GDT is available. */
3137 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3138 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3139 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3140 AssertRCReturn(rc, rc);
3141 }
3142 }
3143#else
3144 NOREF(pVM);
3145 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3146#endif
3147 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3148 AssertRCReturn(rc, rc);
3149
3150 /*
3151 * Host FS base and GS base.
3152 */
3153#if HC_ARCH_BITS == 64
3154 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3155 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3156 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3157 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3158 AssertRCReturn(rc, rc);
3159
3160 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3161 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3162 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3163 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3164 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3165#endif
3166 return rc;
3167}
3168
3169
3170/**
3171 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3172 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3173 * the host after every successful VM-exit.
3174 *
3175 * @returns VBox status code.
3176 * @param pVM The cross context VM structure.
3177 * @param pVCpu The cross context virtual CPU structure.
3178 *
3179 * @remarks No-long-jump zone!!!
3180 */
3181DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3182{
3183 NOREF(pVM);
3184
3185 AssertPtr(pVCpu);
3186 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3187
3188 /*
3189 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3190 * rather than swapping them on every VM-entry.
3191 */
3192 hmR0VmxLazySaveHostMsrs(pVCpu);
3193
3194 /*
3195 * Host Sysenter MSRs.
3196 */
3197 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3198#if HC_ARCH_BITS == 32
3199 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3200 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3201#else
3202 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3203 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3204#endif
3205 AssertRCReturn(rc, rc);
3206
3207 /*
3208 * Host EFER MSR.
3209 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3210 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3211 */
3212 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3213 {
3214 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3215 AssertRCReturn(rc, rc);
3216 }
3217
3218 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3219 * hmR0VmxLoadGuestExitCtls() !! */
3220
3221 return rc;
3222}
3223
3224
3225/**
3226 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3227 *
3228 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3229 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3230 * hmR0VMxLoadGuestEntryCtls().
3231 *
3232 * @returns true if we need to load guest EFER, false otherwise.
3233 * @param pVCpu The cross context virtual CPU structure.
3234 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3235 * out-of-sync. Make sure to update the required fields
3236 * before using them.
3237 *
3238 * @remarks Requires EFER, CR4.
3239 * @remarks No-long-jump zone!!!
3240 */
3241static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3242{
3243#ifdef HMVMX_ALWAYS_SWAP_EFER
3244 return true;
3245#endif
3246
3247#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3248 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3249 if (CPUMIsGuestInLongMode(pVCpu))
3250 return false;
3251#endif
3252
3253 PVM pVM = pVCpu->CTX_SUFF(pVM);
3254 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3255 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3256
3257 /*
3258 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3259 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3260 */
3261 if ( CPUMIsGuestInLongMode(pVCpu)
3262 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3263 {
3264 return true;
3265 }
3266
3267 /*
3268 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3269 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3270 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3271 */
3272 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3273 && (pMixedCtx->cr0 & X86_CR0_PG)
3274 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3275 {
3276 /* Assert that host is PAE capable. */
3277 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3278 return true;
3279 }
3280
3281 /** @todo Check the latest Intel spec. for any other bits,
3282 * like SMEP/SMAP? */
3283 return false;
3284}
3285
3286
3287/**
3288 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3289 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3290 * controls".
3291 *
3292 * @returns VBox status code.
3293 * @param pVCpu The cross context virtual CPU structure.
3294 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3295 * out-of-sync. Make sure to update the required fields
3296 * before using them.
3297 *
3298 * @remarks Requires EFER.
3299 * @remarks No-long-jump zone!!!
3300 */
3301DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3302{
3303 int rc = VINF_SUCCESS;
3304 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3305 {
3306 PVM pVM = pVCpu->CTX_SUFF(pVM);
3307 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3308 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3309
3310 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3311 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3312
3313 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3314 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3315 {
3316 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3317 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3318 }
3319 else
3320 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3321
3322 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3323 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3324 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3325 {
3326 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3327 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3328 }
3329
3330 /*
3331 * The following should -not- be set (since we're not in SMM mode):
3332 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3333 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3334 */
3335
3336 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3337 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3338
3339 if ((val & zap) != val)
3340 {
3341 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3342 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3343 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3344 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3345 }
3346
3347 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3348 AssertRCReturn(rc, rc);
3349
3350 pVCpu->hm.s.vmx.u32EntryCtls = val;
3351 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3352 }
3353 return rc;
3354}
3355
3356
3357/**
3358 * Sets up the VM-exit controls in the VMCS.
3359 *
3360 * @returns VBox status code.
3361 * @param pVCpu The cross context virtual CPU structure.
3362 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3363 * out-of-sync. Make sure to update the required fields
3364 * before using them.
3365 *
3366 * @remarks Requires EFER.
3367 */
3368DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3369{
3370 NOREF(pMixedCtx);
3371
3372 int rc = VINF_SUCCESS;
3373 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3374 {
3375 PVM pVM = pVCpu->CTX_SUFF(pVM);
3376 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3377 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3378
3379 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3380 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3381
3382 /*
3383 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3384 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3385 */
3386#if HC_ARCH_BITS == 64
3387 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3388 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3389#else
3390 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3391 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3392 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3393 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3394 {
3395 /* The switcher returns to long mode, EFER is managed by the switcher. */
3396 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3397 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3398 }
3399 else
3400 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3401#endif
3402
3403 /* If the newer VMCS fields for managing EFER exists, use it. */
3404 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3405 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3406 {
3407 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3408 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3409 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3410 }
3411
3412 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3413 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3414
3415 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3416 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3417 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3418
3419 if ( pVM->hm.s.vmx.fUsePreemptTimer
3420 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3421 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3422
3423 if ((val & zap) != val)
3424 {
3425 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3426 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3427 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3428 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3429 }
3430
3431 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3432 AssertRCReturn(rc, rc);
3433
3434 pVCpu->hm.s.vmx.u32ExitCtls = val;
3435 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3436 }
3437 return rc;
3438}
3439
3440
3441/**
3442 * Sets the TPR threshold in the VMCS.
3443 *
3444 * @returns VBox status code.
3445 * @param pVCpu The cross context virtual CPU structure.
3446 * @param u32TprThreshold The TPR threshold (task-priority class only).
3447 */
3448DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3449{
3450 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3451 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3452 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3453}
3454
3455
3456/**
3457 * Loads the guest APIC and related state.
3458 *
3459 * @returns VBox status code.
3460 * @param pVCpu The cross context virtual CPU structure.
3461 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3462 * out-of-sync. Make sure to update the required fields
3463 * before using them.
3464 *
3465 * @remarks No-long-jump zone!!!
3466 */
3467DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3468{
3469 NOREF(pMixedCtx);
3470
3471 int rc = VINF_SUCCESS;
3472 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3473 {
3474 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3475 && APICIsEnabled(pVCpu))
3476 {
3477 /*
3478 * Setup TPR shadowing.
3479 */
3480 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3481 {
3482 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3483
3484 bool fPendingIntr = false;
3485 uint8_t u8Tpr = 0;
3486 uint8_t u8PendingIntr = 0;
3487 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3488 AssertRCReturn(rc, rc);
3489
3490 /*
3491 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3492 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3493 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3494 */
3495 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3496 uint32_t u32TprThreshold = 0;
3497 if (fPendingIntr)
3498 {
3499 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3500 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3501 const uint8_t u8TprPriority = u8Tpr >> 4;
3502 if (u8PendingPriority <= u8TprPriority)
3503 u32TprThreshold = u8PendingPriority;
3504 }
3505
3506 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3507 AssertRCReturn(rc, rc);
3508 }
3509 }
3510 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3511 }
3512
3513 return rc;
3514}
3515
3516
3517/**
3518 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3519 *
3520 * @returns Guest's interruptibility-state.
3521 * @param pVCpu The cross context virtual CPU structure.
3522 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3523 * out-of-sync. Make sure to update the required fields
3524 * before using them.
3525 *
3526 * @remarks No-long-jump zone!!!
3527 */
3528DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3529{
3530 /*
3531 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3532 */
3533 uint32_t uIntrState = 0;
3534 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3535 {
3536 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3537 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3538 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3539 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3540 {
3541 if (pMixedCtx->eflags.Bits.u1IF)
3542 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3543 else
3544 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3545 }
3546 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3547 {
3548 /*
3549 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3550 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3551 */
3552 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3553 }
3554 }
3555
3556 /*
3557 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3558 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3559 * setting this would block host-NMIs and IRET will not clear the blocking.
3560 *
3561 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3562 */
3563 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3564 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3565 {
3566 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3567 }
3568
3569 return uIntrState;
3570}
3571
3572
3573/**
3574 * Loads the guest's interruptibility-state into the guest-state area in the
3575 * VMCS.
3576 *
3577 * @returns VBox status code.
3578 * @param pVCpu The cross context virtual CPU structure.
3579 * @param uIntrState The interruptibility-state to set.
3580 */
3581static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3582{
3583 NOREF(pVCpu);
3584 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3585 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3586 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3587 AssertRC(rc);
3588 return rc;
3589}
3590
3591
3592/**
3593 * Loads the exception intercepts required for guest execution in the VMCS.
3594 *
3595 * @returns VBox status code.
3596 * @param pVCpu The cross context virtual CPU structure.
3597 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3598 * out-of-sync. Make sure to update the required fields
3599 * before using them.
3600 */
3601static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3602{
3603 NOREF(pMixedCtx);
3604 int rc = VINF_SUCCESS;
3605 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3606 {
3607 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3608 if (pVCpu->hm.s.fGIMTrapXcptUD)
3609 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3610#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3611 else
3612 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3613#endif
3614
3615 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3616 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3617
3618 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3619 AssertRCReturn(rc, rc);
3620
3621 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3622 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3623 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3624 }
3625 return rc;
3626}
3627
3628
3629/**
3630 * Loads the guest's RIP into the guest-state area in the VMCS.
3631 *
3632 * @returns VBox status code.
3633 * @param pVCpu The cross context virtual CPU structure.
3634 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3635 * out-of-sync. Make sure to update the required fields
3636 * before using them.
3637 *
3638 * @remarks No-long-jump zone!!!
3639 */
3640static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3641{
3642 int rc = VINF_SUCCESS;
3643 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3644 {
3645 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3646 AssertRCReturn(rc, rc);
3647
3648 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3649 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3650 HMCPU_CF_VALUE(pVCpu)));
3651
3652 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3653 if (HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
3654 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3655 else
3656 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3657 }
3658 return rc;
3659}
3660
3661
3662/**
3663 * Loads the guest's RSP into the guest-state area in the VMCS.
3664 *
3665 * @returns VBox status code.
3666 * @param pVCpu The cross context virtual CPU structure.
3667 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3668 * out-of-sync. Make sure to update the required fields
3669 * before using them.
3670 *
3671 * @remarks No-long-jump zone!!!
3672 */
3673static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3674{
3675 int rc = VINF_SUCCESS;
3676 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3677 {
3678 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3679 AssertRCReturn(rc, rc);
3680
3681 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3682 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3683 }
3684 return rc;
3685}
3686
3687
3688/**
3689 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3690 *
3691 * @returns VBox status code.
3692 * @param pVCpu The cross context virtual CPU structure.
3693 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3694 * out-of-sync. Make sure to update the required fields
3695 * before using them.
3696 *
3697 * @remarks No-long-jump zone!!!
3698 */
3699static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3700{
3701 int rc = VINF_SUCCESS;
3702 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3703 {
3704 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3705 Let us assert it as such and use 32-bit VMWRITE. */
3706 Assert(!(pMixedCtx->rflags.u64 >> 32));
3707 X86EFLAGS Eflags = pMixedCtx->eflags;
3708 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3709 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3710 * These will never be cleared/set, unless some other part of the VMM
3711 * code is buggy - in which case we're better of finding and fixing
3712 * those bugs than hiding them. */
3713 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3714 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3715 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3716 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3717
3718 /*
3719 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3720 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3721 */
3722 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3723 {
3724 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3725 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3726 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3727 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3728 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3729 }
3730
3731 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3732 AssertRCReturn(rc, rc);
3733
3734 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3735 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3736 }
3737 return rc;
3738}
3739
3740
3741/**
3742 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3743 *
3744 * @returns VBox status code.
3745 * @param pVCpu The cross context virtual CPU structure.
3746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3747 * out-of-sync. Make sure to update the required fields
3748 * before using them.
3749 *
3750 * @remarks No-long-jump zone!!!
3751 */
3752DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3753{
3754 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3755 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3756 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3757 AssertRCReturn(rc, rc);
3758 return rc;
3759}
3760
3761
3762/**
3763 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3764 * CR0 is partially shared with the host and we have to consider the FPU bits.
3765 *
3766 * @returns VBox status code.
3767 * @param pVCpu The cross context virtual CPU structure.
3768 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3769 * out-of-sync. Make sure to update the required fields
3770 * before using them.
3771 *
3772 * @remarks No-long-jump zone!!!
3773 */
3774static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3775{
3776 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3777
3778 /*
3779 * Guest CR0.
3780 * Guest FPU.
3781 */
3782 int rc = VINF_SUCCESS;
3783 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3784 {
3785 Assert(!(pMixedCtx->cr0 >> 32));
3786 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3787 PVM pVM = pVCpu->CTX_SUFF(pVM);
3788
3789 /* The guest's view (read access) of its CR0 is unblemished. */
3790 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3791 AssertRCReturn(rc, rc);
3792 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3793
3794 /* Setup VT-x's view of the guest CR0. */
3795 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3796 if (pVM->hm.s.fNestedPaging)
3797 {
3798 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3799 {
3800 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3801 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3802 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3803 }
3804 else
3805 {
3806 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3807 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3808 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3809 }
3810
3811 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3812 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3813 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3814
3815 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3816 AssertRCReturn(rc, rc);
3817 }
3818 else
3819 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3820
3821 /*
3822 * Guest FPU bits.
3823 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3824 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3825 */
3826 u32GuestCR0 |= X86_CR0_NE;
3827
3828 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3829 bool fInterceptMF = false;
3830 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3831 fInterceptMF = true;
3832
3833 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3834 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3835 {
3836 Assert(PDMVmmDevHeapIsEnabled(pVM));
3837 Assert(pVM->hm.s.vmx.pRealModeTSS);
3838 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3839 }
3840 else
3841 {
3842 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3843 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3844 if (fInterceptMF)
3845 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3846 }
3847 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3848
3849 /* Additional intercepts for debugging, define these yourself explicitly. */
3850#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3851 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3852 | RT_BIT(X86_XCPT_BP)
3853 | RT_BIT(X86_XCPT_DE)
3854 | RT_BIT(X86_XCPT_NM)
3855 | RT_BIT(X86_XCPT_TS)
3856 | RT_BIT(X86_XCPT_UD)
3857 | RT_BIT(X86_XCPT_NP)
3858 | RT_BIT(X86_XCPT_SS)
3859 | RT_BIT(X86_XCPT_GP)
3860 | RT_BIT(X86_XCPT_PF)
3861 | RT_BIT(X86_XCPT_MF)
3862 ;
3863#elif defined(HMVMX_ALWAYS_TRAP_PF)
3864 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3865#endif
3866
3867 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3868
3869 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3870 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3871 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3872 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3873 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3874 else
3875 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3876
3877 u32GuestCR0 |= uSetCR0;
3878 u32GuestCR0 &= uZapCR0;
3879 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3880
3881 /* Write VT-x's view of the guest CR0 into the VMCS. */
3882 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3883 AssertRCReturn(rc, rc);
3884 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3885 uZapCR0));
3886
3887 /*
3888 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3889 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3890 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3891 */
3892 uint32_t u32CR0Mask = 0;
3893 u32CR0Mask = X86_CR0_PE
3894 | X86_CR0_NE
3895 | X86_CR0_WP
3896 | X86_CR0_PG
3897 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3898 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3899 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3900
3901 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3902 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3903 * and @bugref{6944}. */
3904#if 0
3905 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3906 u32CR0Mask &= ~X86_CR0_PE;
3907#endif
3908 if (pVM->hm.s.fNestedPaging)
3909 u32CR0Mask &= ~X86_CR0_WP;
3910
3911 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3912 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3913 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3914 AssertRCReturn(rc, rc);
3915 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3916
3917 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3918 }
3919 return rc;
3920}
3921
3922
3923/**
3924 * Loads the guest control registers (CR3, CR4) into the guest-state area
3925 * in the VMCS.
3926 *
3927 * @returns VBox strict status code.
3928 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3929 * without unrestricted guest access and the VMMDev is not presently
3930 * mapped (e.g. EFI32).
3931 *
3932 * @param pVCpu The cross context virtual CPU structure.
3933 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3934 * out-of-sync. Make sure to update the required fields
3935 * before using them.
3936 *
3937 * @remarks No-long-jump zone!!!
3938 */
3939static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3940{
3941 int rc = VINF_SUCCESS;
3942 PVM pVM = pVCpu->CTX_SUFF(pVM);
3943
3944 /*
3945 * Guest CR2.
3946 * It's always loaded in the assembler code. Nothing to do here.
3947 */
3948
3949 /*
3950 * Guest CR3.
3951 */
3952 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3953 {
3954 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3955 if (pVM->hm.s.fNestedPaging)
3956 {
3957 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3958
3959 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3960 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3961 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3962 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3963
3964 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3965 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3966 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3967
3968 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3969 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3970 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3971 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3972 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3973 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3974 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3975
3976 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3977 AssertRCReturn(rc, rc);
3978 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3979
3980 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3981 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3982 {
3983 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3984 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3985 {
3986 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3987 AssertRCReturn(rc, rc);
3988 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3989 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3990 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3991 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3992 AssertRCReturn(rc, rc);
3993 }
3994
3995 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3996 have Unrestricted Execution to handle the guest when it's not using paging. */
3997 GCPhysGuestCR3 = pMixedCtx->cr3;
3998 }
3999 else
4000 {
4001 /*
4002 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4003 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4004 * EPT takes care of translating it to host-physical addresses.
4005 */
4006 RTGCPHYS GCPhys;
4007 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4008
4009 /* We obtain it here every time as the guest could have relocated this PCI region. */
4010 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4011 if (RT_SUCCESS(rc))
4012 { /* likely */ }
4013 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4014 {
4015 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4016 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4017 }
4018 else
4019 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4020
4021 GCPhysGuestCR3 = GCPhys;
4022 }
4023
4024 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4025 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4026 }
4027 else
4028 {
4029 /* Non-nested paging case, just use the hypervisor's CR3. */
4030 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4031
4032 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4033 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4034 }
4035 AssertRCReturn(rc, rc);
4036
4037 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4038 }
4039
4040 /*
4041 * Guest CR4.
4042 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4043 */
4044 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4045 {
4046 Assert(!(pMixedCtx->cr4 >> 32));
4047 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4048
4049 /* The guest's view of its CR4 is unblemished. */
4050 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4051 AssertRCReturn(rc, rc);
4052 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4053
4054 /* Setup VT-x's view of the guest CR4. */
4055 /*
4056 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4057 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4058 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4059 */
4060 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4061 {
4062 Assert(pVM->hm.s.vmx.pRealModeTSS);
4063 Assert(PDMVmmDevHeapIsEnabled(pVM));
4064 u32GuestCR4 &= ~X86_CR4_VME;
4065 }
4066
4067 if (pVM->hm.s.fNestedPaging)
4068 {
4069 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4070 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4071 {
4072 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4073 u32GuestCR4 |= X86_CR4_PSE;
4074 /* Our identity mapping is a 32-bit page directory. */
4075 u32GuestCR4 &= ~X86_CR4_PAE;
4076 }
4077 /* else use guest CR4.*/
4078 }
4079 else
4080 {
4081 /*
4082 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4083 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4084 */
4085 switch (pVCpu->hm.s.enmShadowMode)
4086 {
4087 case PGMMODE_REAL: /* Real-mode. */
4088 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4089 case PGMMODE_32_BIT: /* 32-bit paging. */
4090 {
4091 u32GuestCR4 &= ~X86_CR4_PAE;
4092 break;
4093 }
4094
4095 case PGMMODE_PAE: /* PAE paging. */
4096 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4097 {
4098 u32GuestCR4 |= X86_CR4_PAE;
4099 break;
4100 }
4101
4102 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4103 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4104#ifdef VBOX_ENABLE_64_BITS_GUESTS
4105 break;
4106#endif
4107 default:
4108 AssertFailed();
4109 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4110 }
4111 }
4112
4113 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4114 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4115 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4116 u32GuestCR4 |= uSetCR4;
4117 u32GuestCR4 &= uZapCR4;
4118
4119 /* Write VT-x's view of the guest CR4 into the VMCS. */
4120 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4121 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4122 AssertRCReturn(rc, rc);
4123
4124 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4125 uint32_t u32CR4Mask = X86_CR4_VME
4126 | X86_CR4_PAE
4127 | X86_CR4_PGE
4128 | X86_CR4_PSE
4129 | X86_CR4_VMXE;
4130 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4131 u32CR4Mask |= X86_CR4_OSXSAVE;
4132 if (pVM->cpum.ro.GuestFeatures.fPcid)
4133 u32CR4Mask |= X86_CR4_PCIDE;
4134 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4135 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4136 AssertRCReturn(rc, rc);
4137
4138 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4139 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4140
4141 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4142 }
4143 return rc;
4144}
4145
4146
4147/**
4148 * Loads the guest debug registers into the guest-state area in the VMCS.
4149 *
4150 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4151 *
4152 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4153 *
4154 * @returns VBox status code.
4155 * @param pVCpu The cross context virtual CPU structure.
4156 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4157 * out-of-sync. Make sure to update the required fields
4158 * before using them.
4159 *
4160 * @remarks No-long-jump zone!!!
4161 */
4162static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4163{
4164 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4165 return VINF_SUCCESS;
4166
4167#ifdef VBOX_STRICT
4168 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4169 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4170 {
4171 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4172 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4173 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4174 }
4175#endif
4176
4177 int rc;
4178 PVM pVM = pVCpu->CTX_SUFF(pVM);
4179 bool fSteppingDB = false;
4180 bool fInterceptMovDRx = false;
4181 if (pVCpu->hm.s.fSingleInstruction)
4182 {
4183 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4184 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4185 {
4186 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4187 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4188 AssertRCReturn(rc, rc);
4189 Assert(fSteppingDB == false);
4190 }
4191 else
4192 {
4193 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4194 pVCpu->hm.s.fClearTrapFlag = true;
4195 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4196 fSteppingDB = true;
4197 }
4198 }
4199
4200 if ( fSteppingDB
4201 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4202 {
4203 /*
4204 * Use the combined guest and host DRx values found in the hypervisor
4205 * register set because the debugger has breakpoints active or someone
4206 * is single stepping on the host side without a monitor trap flag.
4207 *
4208 * Note! DBGF expects a clean DR6 state before executing guest code.
4209 */
4210#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4211 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4212 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4213 {
4214 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4215 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4216 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4217 }
4218 else
4219#endif
4220 if (!CPUMIsHyperDebugStateActive(pVCpu))
4221 {
4222 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4223 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4224 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4225 }
4226
4227 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4228 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4229 AssertRCReturn(rc, rc);
4230
4231 pVCpu->hm.s.fUsingHyperDR7 = true;
4232 fInterceptMovDRx = true;
4233 }
4234 else
4235 {
4236 /*
4237 * If the guest has enabled debug registers, we need to load them prior to
4238 * executing guest code so they'll trigger at the right time.
4239 */
4240 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4241 {
4242#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4243 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4244 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4245 {
4246 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4247 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4248 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4249 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4250 }
4251 else
4252#endif
4253 if (!CPUMIsGuestDebugStateActive(pVCpu))
4254 {
4255 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4256 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4257 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4258 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4259 }
4260 Assert(!fInterceptMovDRx);
4261 }
4262 /*
4263 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4264 * must intercept #DB in order to maintain a correct DR6 guest value, and
4265 * because we need to intercept it to prevent nested #DBs from hanging the
4266 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4267 */
4268#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4269 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4270 && !CPUMIsGuestDebugStateActive(pVCpu))
4271#else
4272 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4273#endif
4274 {
4275 fInterceptMovDRx = true;
4276 }
4277
4278 /* Update guest DR7. */
4279 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4280 AssertRCReturn(rc, rc);
4281
4282 pVCpu->hm.s.fUsingHyperDR7 = false;
4283 }
4284
4285 /*
4286 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4287 */
4288 if (fInterceptMovDRx)
4289 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4290 else
4291 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4292 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4293 AssertRCReturn(rc, rc);
4294
4295 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4296 return VINF_SUCCESS;
4297}
4298
4299
4300#ifdef VBOX_STRICT
4301/**
4302 * Strict function to validate segment registers.
4303 *
4304 * @remarks ASSUMES CR0 is up to date.
4305 */
4306static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4307{
4308 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4309 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4310 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4311 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4312 && ( !CPUMIsGuestInRealModeEx(pCtx)
4313 && !CPUMIsGuestInV86ModeEx(pCtx)))
4314 {
4315 /* Protected mode checks */
4316 /* CS */
4317 Assert(pCtx->cs.Attr.n.u1Present);
4318 Assert(!(pCtx->cs.Attr.u & 0xf00));
4319 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4320 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4321 || !(pCtx->cs.Attr.n.u1Granularity));
4322 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4323 || (pCtx->cs.Attr.n.u1Granularity));
4324 /* CS cannot be loaded with NULL in protected mode. */
4325 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4326 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4327 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4328 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4329 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4330 else
4331 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4332 /* SS */
4333 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4334 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4335 if ( !(pCtx->cr0 & X86_CR0_PE)
4336 || pCtx->cs.Attr.n.u4Type == 3)
4337 {
4338 Assert(!pCtx->ss.Attr.n.u2Dpl);
4339 }
4340 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4341 {
4342 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4343 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4344 Assert(pCtx->ss.Attr.n.u1Present);
4345 Assert(!(pCtx->ss.Attr.u & 0xf00));
4346 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4347 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4348 || !(pCtx->ss.Attr.n.u1Granularity));
4349 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4350 || (pCtx->ss.Attr.n.u1Granularity));
4351 }
4352 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4353 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4354 {
4355 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4356 Assert(pCtx->ds.Attr.n.u1Present);
4357 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4358 Assert(!(pCtx->ds.Attr.u & 0xf00));
4359 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4360 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4361 || !(pCtx->ds.Attr.n.u1Granularity));
4362 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4363 || (pCtx->ds.Attr.n.u1Granularity));
4364 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4365 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4366 }
4367 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4368 {
4369 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4370 Assert(pCtx->es.Attr.n.u1Present);
4371 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4372 Assert(!(pCtx->es.Attr.u & 0xf00));
4373 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4374 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4375 || !(pCtx->es.Attr.n.u1Granularity));
4376 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4377 || (pCtx->es.Attr.n.u1Granularity));
4378 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4379 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4380 }
4381 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4382 {
4383 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4384 Assert(pCtx->fs.Attr.n.u1Present);
4385 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4386 Assert(!(pCtx->fs.Attr.u & 0xf00));
4387 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4388 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4389 || !(pCtx->fs.Attr.n.u1Granularity));
4390 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4391 || (pCtx->fs.Attr.n.u1Granularity));
4392 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4393 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4394 }
4395 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4396 {
4397 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4398 Assert(pCtx->gs.Attr.n.u1Present);
4399 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4400 Assert(!(pCtx->gs.Attr.u & 0xf00));
4401 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4402 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4403 || !(pCtx->gs.Attr.n.u1Granularity));
4404 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4405 || (pCtx->gs.Attr.n.u1Granularity));
4406 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4407 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4408 }
4409 /* 64-bit capable CPUs. */
4410# if HC_ARCH_BITS == 64
4411 Assert(!(pCtx->cs.u64Base >> 32));
4412 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4413 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4414 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4415# endif
4416 }
4417 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4418 || ( CPUMIsGuestInRealModeEx(pCtx)
4419 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4420 {
4421 /* Real and v86 mode checks. */
4422 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4423 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4424 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4425 {
4426 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4427 }
4428 else
4429 {
4430 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4431 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4432 }
4433
4434 /* CS */
4435 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4436 Assert(pCtx->cs.u32Limit == 0xffff);
4437 Assert(u32CSAttr == 0xf3);
4438 /* SS */
4439 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4440 Assert(pCtx->ss.u32Limit == 0xffff);
4441 Assert(u32SSAttr == 0xf3);
4442 /* DS */
4443 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4444 Assert(pCtx->ds.u32Limit == 0xffff);
4445 Assert(u32DSAttr == 0xf3);
4446 /* ES */
4447 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4448 Assert(pCtx->es.u32Limit == 0xffff);
4449 Assert(u32ESAttr == 0xf3);
4450 /* FS */
4451 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4452 Assert(pCtx->fs.u32Limit == 0xffff);
4453 Assert(u32FSAttr == 0xf3);
4454 /* GS */
4455 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4456 Assert(pCtx->gs.u32Limit == 0xffff);
4457 Assert(u32GSAttr == 0xf3);
4458 /* 64-bit capable CPUs. */
4459# if HC_ARCH_BITS == 64
4460 Assert(!(pCtx->cs.u64Base >> 32));
4461 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4462 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4463 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4464# endif
4465 }
4466}
4467#endif /* VBOX_STRICT */
4468
4469
4470/**
4471 * Writes a guest segment register into the guest-state area in the VMCS.
4472 *
4473 * @returns VBox status code.
4474 * @param pVCpu The cross context virtual CPU structure.
4475 * @param idxSel Index of the selector in the VMCS.
4476 * @param idxLimit Index of the segment limit in the VMCS.
4477 * @param idxBase Index of the segment base in the VMCS.
4478 * @param idxAccess Index of the access rights of the segment in the VMCS.
4479 * @param pSelReg Pointer to the segment selector.
4480 *
4481 * @remarks No-long-jump zone!!!
4482 */
4483static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4484 uint32_t idxAccess, PCPUMSELREG pSelReg)
4485{
4486 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4487 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4488 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4489 AssertRCReturn(rc, rc);
4490
4491 uint32_t u32Access = pSelReg->Attr.u;
4492 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4493 {
4494 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4495 u32Access = 0xf3;
4496 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4497 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4498 }
4499 else
4500 {
4501 /*
4502 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4503 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4504 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4505 * loaded in protected-mode have their attribute as 0.
4506 */
4507 if (!u32Access)
4508 u32Access = X86DESCATTR_UNUSABLE;
4509 }
4510
4511 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4512 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4513 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4514
4515 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4516 AssertRCReturn(rc, rc);
4517 return rc;
4518}
4519
4520
4521/**
4522 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4523 * into the guest-state area in the VMCS.
4524 *
4525 * @returns VBox status code.
4526 * @param pVCpu The cross context virtual CPU structure.
4527 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4528 * out-of-sync. Make sure to update the required fields
4529 * before using them.
4530 *
4531 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4532 * @remarks No-long-jump zone!!!
4533 */
4534static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4535{
4536 int rc = VERR_INTERNAL_ERROR_5;
4537 PVM pVM = pVCpu->CTX_SUFF(pVM);
4538
4539 /*
4540 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4541 */
4542 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4543 {
4544 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4545 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4546 {
4547 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4548 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4549 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4550 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4551 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4552 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4553 }
4554
4555#ifdef VBOX_WITH_REM
4556 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4557 {
4558 Assert(pVM->hm.s.vmx.pRealModeTSS);
4559 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4560 if ( pVCpu->hm.s.vmx.fWasInRealMode
4561 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4562 {
4563 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4564 in real-mode (e.g. OpenBSD 4.0) */
4565 REMFlushTBs(pVM);
4566 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4567 pVCpu->hm.s.vmx.fWasInRealMode = false;
4568 }
4569 }
4570#endif
4571 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4572 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4573 AssertRCReturn(rc, rc);
4574 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4575 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4576 AssertRCReturn(rc, rc);
4577 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4578 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4579 AssertRCReturn(rc, rc);
4580 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4581 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4582 AssertRCReturn(rc, rc);
4583 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4584 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4585 AssertRCReturn(rc, rc);
4586 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4587 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4588 AssertRCReturn(rc, rc);
4589
4590#ifdef VBOX_STRICT
4591 /* Validate. */
4592 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4593#endif
4594
4595 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4596 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4597 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4598
4599 /* Update the exit history entry with the correct CS.BASE + RIP. */
4600 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
4601 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4602 }
4603
4604 /*
4605 * Guest TR.
4606 */
4607 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4608 {
4609 /*
4610 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4611 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4612 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4613 */
4614 uint16_t u16Sel = 0;
4615 uint32_t u32Limit = 0;
4616 uint64_t u64Base = 0;
4617 uint32_t u32AccessRights = 0;
4618
4619 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4620 {
4621 u16Sel = pMixedCtx->tr.Sel;
4622 u32Limit = pMixedCtx->tr.u32Limit;
4623 u64Base = pMixedCtx->tr.u64Base;
4624 u32AccessRights = pMixedCtx->tr.Attr.u;
4625 }
4626 else
4627 {
4628 Assert(pVM->hm.s.vmx.pRealModeTSS);
4629 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4630
4631 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4632 RTGCPHYS GCPhys;
4633 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4634 AssertRCReturn(rc, rc);
4635
4636 X86DESCATTR DescAttr;
4637 DescAttr.u = 0;
4638 DescAttr.n.u1Present = 1;
4639 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4640
4641 u16Sel = 0;
4642 u32Limit = HM_VTX_TSS_SIZE;
4643 u64Base = GCPhys; /* in real-mode phys = virt. */
4644 u32AccessRights = DescAttr.u;
4645 }
4646
4647 /* Validate. */
4648 Assert(!(u16Sel & RT_BIT(2)));
4649 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4650 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4651 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4652 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4653 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4654 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4655 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4656 Assert( (u32Limit & 0xfff) == 0xfff
4657 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4658 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4659 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4660
4661 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4662 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4663 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4664 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4665 AssertRCReturn(rc, rc);
4666
4667 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4668 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4669 }
4670
4671 /*
4672 * Guest GDTR.
4673 */
4674 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4675 {
4676 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4677 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4678 AssertRCReturn(rc, rc);
4679
4680 /* Validate. */
4681 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4682
4683 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4684 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4685 }
4686
4687 /*
4688 * Guest LDTR.
4689 */
4690 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4691 {
4692 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4693 uint32_t u32Access = 0;
4694 if (!pMixedCtx->ldtr.Attr.u)
4695 u32Access = X86DESCATTR_UNUSABLE;
4696 else
4697 u32Access = pMixedCtx->ldtr.Attr.u;
4698
4699 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4700 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4701 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4702 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4703 AssertRCReturn(rc, rc);
4704
4705 /* Validate. */
4706 if (!(u32Access & X86DESCATTR_UNUSABLE))
4707 {
4708 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4709 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4710 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4711 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4712 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4713 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4714 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4715 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4716 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4717 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4718 }
4719
4720 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4721 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4722 }
4723
4724 /*
4725 * Guest IDTR.
4726 */
4727 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4728 {
4729 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4730 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4731 AssertRCReturn(rc, rc);
4732
4733 /* Validate. */
4734 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4735
4736 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4737 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4738 }
4739
4740 return VINF_SUCCESS;
4741}
4742
4743
4744/**
4745 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4746 * areas.
4747 *
4748 * These MSRs will automatically be loaded to the host CPU on every successful
4749 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4750 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4751 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4752 *
4753 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4754 *
4755 * @returns VBox status code.
4756 * @param pVCpu The cross context virtual CPU structure.
4757 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4758 * out-of-sync. Make sure to update the required fields
4759 * before using them.
4760 *
4761 * @remarks No-long-jump zone!!!
4762 */
4763static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4764{
4765 AssertPtr(pVCpu);
4766 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4767
4768 /*
4769 * MSRs that we use the auto-load/store MSR area in the VMCS.
4770 */
4771 PVM pVM = pVCpu->CTX_SUFF(pVM);
4772 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4773 {
4774 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4775#if HC_ARCH_BITS == 32
4776 if (pVM->hm.s.fAllow64BitGuests)
4777 {
4778 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4779 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4780 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4781 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4782 AssertRCReturn(rc, rc);
4783# ifdef LOG_ENABLED
4784 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4785 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4786 {
4787 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4788 pMsr->u64Value));
4789 }
4790# endif
4791 }
4792#endif
4793 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4794 }
4795
4796 /*
4797 * Guest Sysenter MSRs.
4798 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4799 * VM-exits on WRMSRs for these MSRs.
4800 */
4801 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4802 {
4803 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4804 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4805 }
4806
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4808 {
4809 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4810 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4811 }
4812
4813 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4814 {
4815 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4816 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4817 }
4818
4819 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4820 {
4821 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4822 {
4823 /*
4824 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4825 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4826 */
4827 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4828 {
4829 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4830 AssertRCReturn(rc,rc);
4831 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4832 }
4833 else
4834 {
4835 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4836 NULL /* pfAddedAndUpdated */);
4837 AssertRCReturn(rc, rc);
4838
4839 /* We need to intercept reads too, see @bugref{7386#c16}. */
4840 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4841 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4842 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4843 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4844 }
4845 }
4846 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4847 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4848 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4849 }
4850
4851 return VINF_SUCCESS;
4852}
4853
4854
4855/**
4856 * Loads the guest activity state into the guest-state area in the VMCS.
4857 *
4858 * @returns VBox status code.
4859 * @param pVCpu The cross context virtual CPU structure.
4860 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4861 * out-of-sync. Make sure to update the required fields
4862 * before using them.
4863 *
4864 * @remarks No-long-jump zone!!!
4865 */
4866static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4867{
4868 NOREF(pMixedCtx);
4869 /** @todo See if we can make use of other states, e.g.
4870 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4871 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4872 {
4873 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4874 AssertRCReturn(rc, rc);
4875
4876 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4877 }
4878 return VINF_SUCCESS;
4879}
4880
4881
4882#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4883/**
4884 * Check if guest state allows safe use of 32-bit switcher again.
4885 *
4886 * Segment bases and protected mode structures must be 32-bit addressable
4887 * because the 32-bit switcher will ignore high dword when writing these VMCS
4888 * fields. See @bugref{8432} for details.
4889 *
4890 * @returns true if safe, false if must continue to use the 64-bit switcher.
4891 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4892 * out-of-sync. Make sure to update the required fields
4893 * before using them.
4894 *
4895 * @remarks No-long-jump zone!!!
4896 */
4897static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4898{
4899 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4900 return false;
4901 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4902 return false;
4903 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4904 return false;
4905 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4906 return false;
4907 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4908 return false;
4909 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4910 return false;
4911 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4912 return false;
4913 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4914 return false;
4915 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4916 return false;
4917 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4918 return false;
4919 /* All good, bases are 32-bit. */
4920 return true;
4921}
4922#endif
4923
4924
4925/**
4926 * Sets up the appropriate function to run guest code.
4927 *
4928 * @returns VBox status code.
4929 * @param pVCpu The cross context virtual CPU structure.
4930 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4931 * out-of-sync. Make sure to update the required fields
4932 * before using them.
4933 *
4934 * @remarks No-long-jump zone!!!
4935 */
4936static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4937{
4938 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4939 {
4940#ifndef VBOX_ENABLE_64_BITS_GUESTS
4941 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4942#endif
4943 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4944#if HC_ARCH_BITS == 32
4945 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4946 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4947 {
4948 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4949 {
4950 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4951 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4952 | HM_CHANGED_VMX_ENTRY_CTLS
4953 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4954 }
4955 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4956
4957 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4958 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4959 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4960 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4961 }
4962#else
4963 /* 64-bit host. */
4964 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4965#endif
4966 }
4967 else
4968 {
4969 /* Guest is not in long mode, use the 32-bit handler. */
4970#if HC_ARCH_BITS == 32
4971 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4972 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4973 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4974 {
4975 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4976 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4977 | HM_CHANGED_VMX_ENTRY_CTLS
4978 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4979 }
4980# ifdef VBOX_ENABLE_64_BITS_GUESTS
4981 /*
4982 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4983 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4984 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4985 * the much faster 32-bit switcher again.
4986 */
4987 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4988 {
4989 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4990 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4991 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4992 }
4993 else
4994 {
4995 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4996 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4997 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4998 {
4999 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
5000 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5001 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
5002 | HM_CHANGED_VMX_ENTRY_CTLS
5003 | HM_CHANGED_VMX_EXIT_CTLS
5004 | HM_CHANGED_HOST_CONTEXT);
5005 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5006 }
5007 }
5008# else
5009 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5010# endif
5011#else
5012 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5013#endif
5014 }
5015 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5016 return VINF_SUCCESS;
5017}
5018
5019
5020/**
5021 * Wrapper for running the guest code in VT-x.
5022 *
5023 * @returns VBox status code, no informational status codes.
5024 * @param pVM The cross context VM structure.
5025 * @param pVCpu The cross context virtual CPU structure.
5026 * @param pCtx Pointer to the guest-CPU context.
5027 *
5028 * @remarks No-long-jump zone!!!
5029 */
5030DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5031{
5032 /*
5033 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5034 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5035 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5036 */
5037 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5038 /** @todo Add stats for resume vs launch. */
5039#ifdef VBOX_WITH_KERNEL_USING_XMM
5040 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5041#else
5042 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5043#endif
5044 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5045 return rc;
5046}
5047
5048
5049/**
5050 * Reports world-switch error and dumps some useful debug info.
5051 *
5052 * @param pVM The cross context VM structure.
5053 * @param pVCpu The cross context virtual CPU structure.
5054 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5055 * @param pCtx Pointer to the guest-CPU context.
5056 * @param pVmxTransient Pointer to the VMX transient structure (only
5057 * exitReason updated).
5058 */
5059static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5060{
5061 Assert(pVM);
5062 Assert(pVCpu);
5063 Assert(pCtx);
5064 Assert(pVmxTransient);
5065 HMVMX_ASSERT_PREEMPT_SAFE();
5066
5067 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5068 switch (rcVMRun)
5069 {
5070 case VERR_VMX_INVALID_VMXON_PTR:
5071 AssertFailed();
5072 break;
5073 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5074 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5075 {
5076 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5077 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5078 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5079 AssertRC(rc);
5080
5081 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5082 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5083 Cannot do it here as we may have been long preempted. */
5084
5085#ifdef VBOX_STRICT
5086 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5087 pVmxTransient->uExitReason));
5088 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5089 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5090 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5091 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5092 else
5093 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5094 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5095 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5096
5097 /* VMX control bits. */
5098 uint32_t u32Val;
5099 uint64_t u64Val;
5100 RTHCUINTREG uHCReg;
5101 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5102 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5103 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5104 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5105 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5106 {
5107 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5108 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5109 }
5110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5113 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5120 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5121 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5124 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5125 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5126 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5127 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5128 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5129 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5130 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5131 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5132 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5133 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5134 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5135 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5137 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5138 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5139 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5140 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5141 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5143 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5144 if (pVM->hm.s.fNestedPaging)
5145 {
5146 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5147 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5148 }
5149
5150 /* Guest bits. */
5151 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5152 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5153 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5154 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5155 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5156 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5157 if (pVM->hm.s.vmx.fVpid)
5158 {
5159 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5160 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5161 }
5162
5163 /* Host bits. */
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5165 Log4(("Host CR0 %#RHr\n", uHCReg));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5167 Log4(("Host CR3 %#RHr\n", uHCReg));
5168 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5169 Log4(("Host CR4 %#RHr\n", uHCReg));
5170
5171 RTGDTR HostGdtr;
5172 PCX86DESCHC pDesc;
5173 ASMGetGDTR(&HostGdtr);
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host CS %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5180 }
5181
5182 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5183 Log4(("Host DS %#08x\n", u32Val));
5184 if (u32Val < HostGdtr.cbGdt)
5185 {
5186 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5187 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5188 }
5189
5190 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5191 Log4(("Host ES %#08x\n", u32Val));
5192 if (u32Val < HostGdtr.cbGdt)
5193 {
5194 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5195 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5196 }
5197
5198 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5199 Log4(("Host FS %#08x\n", u32Val));
5200 if (u32Val < HostGdtr.cbGdt)
5201 {
5202 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5203 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5204 }
5205
5206 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5207 Log4(("Host GS %#08x\n", u32Val));
5208 if (u32Val < HostGdtr.cbGdt)
5209 {
5210 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5211 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5212 }
5213
5214 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5215 Log4(("Host SS %#08x\n", u32Val));
5216 if (u32Val < HostGdtr.cbGdt)
5217 {
5218 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5219 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5220 }
5221
5222 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5223 Log4(("Host TR %#08x\n", u32Val));
5224 if (u32Val < HostGdtr.cbGdt)
5225 {
5226 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5227 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5228 }
5229
5230 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5231 Log4(("Host TR Base %#RHv\n", uHCReg));
5232 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5233 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5234 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5235 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5236 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5237 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5238 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5239 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5240 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5241 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5242 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5243 Log4(("Host RSP %#RHv\n", uHCReg));
5244 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5245 Log4(("Host RIP %#RHv\n", uHCReg));
5246# if HC_ARCH_BITS == 64
5247 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5248 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5249 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5250 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5251 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5252 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5253# endif
5254#endif /* VBOX_STRICT */
5255 break;
5256 }
5257
5258 default:
5259 /* Impossible */
5260 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5261 break;
5262 }
5263 NOREF(pVM); NOREF(pCtx);
5264}
5265
5266
5267#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5268#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5269# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5270#endif
5271#ifdef VBOX_STRICT
5272static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5273{
5274 switch (idxField)
5275 {
5276 case VMX_VMCS_GUEST_RIP:
5277 case VMX_VMCS_GUEST_RSP:
5278 case VMX_VMCS_GUEST_SYSENTER_EIP:
5279 case VMX_VMCS_GUEST_SYSENTER_ESP:
5280 case VMX_VMCS_GUEST_GDTR_BASE:
5281 case VMX_VMCS_GUEST_IDTR_BASE:
5282 case VMX_VMCS_GUEST_CS_BASE:
5283 case VMX_VMCS_GUEST_DS_BASE:
5284 case VMX_VMCS_GUEST_ES_BASE:
5285 case VMX_VMCS_GUEST_FS_BASE:
5286 case VMX_VMCS_GUEST_GS_BASE:
5287 case VMX_VMCS_GUEST_SS_BASE:
5288 case VMX_VMCS_GUEST_LDTR_BASE:
5289 case VMX_VMCS_GUEST_TR_BASE:
5290 case VMX_VMCS_GUEST_CR3:
5291 return true;
5292 }
5293 return false;
5294}
5295
5296static bool hmR0VmxIsValidReadField(uint32_t idxField)
5297{
5298 switch (idxField)
5299 {
5300 /* Read-only fields. */
5301 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5302 return true;
5303 }
5304 /* Remaining readable fields should also be writable. */
5305 return hmR0VmxIsValidWriteField(idxField);
5306}
5307#endif /* VBOX_STRICT */
5308
5309
5310/**
5311 * Executes the specified handler in 64-bit mode.
5312 *
5313 * @returns VBox status code (no informational status codes).
5314 * @param pVM The cross context VM structure.
5315 * @param pVCpu The cross context virtual CPU structure.
5316 * @param pCtx Pointer to the guest CPU context.
5317 * @param enmOp The operation to perform.
5318 * @param cParams Number of parameters.
5319 * @param paParam Array of 32-bit parameters.
5320 */
5321VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5322 uint32_t cParams, uint32_t *paParam)
5323{
5324 NOREF(pCtx);
5325
5326 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5327 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5328 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5329 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5330
5331#ifdef VBOX_STRICT
5332 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5333 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5334
5335 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5336 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5337#endif
5338
5339 /* Disable interrupts. */
5340 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5341
5342#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5343 RTCPUID idHostCpu = RTMpCpuId();
5344 CPUMR0SetLApic(pVCpu, idHostCpu);
5345#endif
5346
5347 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5348 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5349
5350 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5351 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5352 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5353
5354 /* Leave VMX Root Mode. */
5355 VMXDisable();
5356
5357 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5358
5359 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5360 CPUMSetHyperEIP(pVCpu, enmOp);
5361 for (int i = (int)cParams - 1; i >= 0; i--)
5362 CPUMPushHyper(pVCpu, paParam[i]);
5363
5364 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5365
5366 /* Call the switcher. */
5367 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5368 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5369
5370 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5371 /* Make sure the VMX instructions don't cause #UD faults. */
5372 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5373
5374 /* Re-enter VMX Root Mode */
5375 int rc2 = VMXEnable(HCPhysCpuPage);
5376 if (RT_FAILURE(rc2))
5377 {
5378 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5379 ASMSetFlags(fOldEFlags);
5380 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5381 return rc2;
5382 }
5383
5384 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5385 AssertRC(rc2);
5386 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5387 Assert(!(ASMGetFlags() & X86_EFL_IF));
5388 ASMSetFlags(fOldEFlags);
5389 return rc;
5390}
5391
5392
5393/**
5394 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5395 * supporting 64-bit guests.
5396 *
5397 * @returns VBox status code.
5398 * @param fResume Whether to VMLAUNCH or VMRESUME.
5399 * @param pCtx Pointer to the guest-CPU context.
5400 * @param pCache Pointer to the VMCS cache.
5401 * @param pVM The cross context VM structure.
5402 * @param pVCpu The cross context virtual CPU structure.
5403 */
5404DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5405{
5406 NOREF(fResume);
5407
5408 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5409 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5410
5411#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5412 pCache->uPos = 1;
5413 pCache->interPD = PGMGetInterPaeCR3(pVM);
5414 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5415#endif
5416
5417#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5418 pCache->TestIn.HCPhysCpuPage = 0;
5419 pCache->TestIn.HCPhysVmcs = 0;
5420 pCache->TestIn.pCache = 0;
5421 pCache->TestOut.HCPhysVmcs = 0;
5422 pCache->TestOut.pCache = 0;
5423 pCache->TestOut.pCtx = 0;
5424 pCache->TestOut.eflags = 0;
5425#else
5426 NOREF(pCache);
5427#endif
5428
5429 uint32_t aParam[10];
5430 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5431 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5432 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5433 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5434 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5435 aParam[5] = 0;
5436 aParam[6] = VM_RC_ADDR(pVM, pVM);
5437 aParam[7] = 0;
5438 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5439 aParam[9] = 0;
5440
5441#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5442 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5443 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5444#endif
5445 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5446
5447#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5448 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5449 Assert(pCtx->dr[4] == 10);
5450 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5451#endif
5452
5453#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5454 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5455 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5456 pVCpu->hm.s.vmx.HCPhysVmcs));
5457 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5458 pCache->TestOut.HCPhysVmcs));
5459 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5460 pCache->TestOut.pCache));
5461 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5462 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5463 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5464 pCache->TestOut.pCtx));
5465 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5466#endif
5467 return rc;
5468}
5469
5470
5471/**
5472 * Initialize the VMCS-Read cache.
5473 *
5474 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5475 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5476 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5477 * (those that have a 32-bit FULL & HIGH part).
5478 *
5479 * @returns VBox status code.
5480 * @param pVM The cross context VM structure.
5481 * @param pVCpu The cross context virtual CPU structure.
5482 */
5483static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5484{
5485#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5486{ \
5487 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5488 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5489 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5490 ++cReadFields; \
5491}
5492
5493 AssertPtr(pVM);
5494 AssertPtr(pVCpu);
5495 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5496 uint32_t cReadFields = 0;
5497
5498 /*
5499 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5500 * and serve to indicate exceptions to the rules.
5501 */
5502
5503 /* Guest-natural selector base fields. */
5504#if 0
5505 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5508#endif
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5516 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5519 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5521#if 0
5522 /* Unused natural width guest-state fields. */
5523 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5524 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5525#endif
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5528
5529 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5530#if 0
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5534 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5535 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5536 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5538 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5539 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5540#endif
5541
5542 /* Natural width guest-state fields. */
5543 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5544#if 0
5545 /* Currently unused field. */
5546 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5547#endif
5548
5549 if (pVM->hm.s.fNestedPaging)
5550 {
5551 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5552 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5553 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5554 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5555 }
5556 else
5557 {
5558 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5559 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5560 }
5561
5562#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5563 return VINF_SUCCESS;
5564}
5565
5566
5567/**
5568 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5569 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5570 * darwin, running 64-bit guests).
5571 *
5572 * @returns VBox status code.
5573 * @param pVCpu The cross context virtual CPU structure.
5574 * @param idxField The VMCS field encoding.
5575 * @param u64Val 16, 32 or 64-bit value.
5576 */
5577VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5578{
5579 int rc;
5580 switch (idxField)
5581 {
5582 /*
5583 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5584 */
5585 /* 64-bit Control fields. */
5586 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5587 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5588 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5589 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5590 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5591 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5592 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5593 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5594 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5595 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5596 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5597 case VMX_VMCS64_CTRL_EPTP_FULL:
5598 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5599 /* 64-bit Guest-state fields. */
5600 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5601 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5602 case VMX_VMCS64_GUEST_PAT_FULL:
5603 case VMX_VMCS64_GUEST_EFER_FULL:
5604 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5605 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5606 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5607 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5608 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5609 /* 64-bit Host-state fields. */
5610 case VMX_VMCS64_HOST_PAT_FULL:
5611 case VMX_VMCS64_HOST_EFER_FULL:
5612 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5613 {
5614 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5615 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5616 break;
5617 }
5618
5619 /*
5620 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5621 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5622 */
5623 /* Natural-width Guest-state fields. */
5624 case VMX_VMCS_GUEST_CR3:
5625 case VMX_VMCS_GUEST_ES_BASE:
5626 case VMX_VMCS_GUEST_CS_BASE:
5627 case VMX_VMCS_GUEST_SS_BASE:
5628 case VMX_VMCS_GUEST_DS_BASE:
5629 case VMX_VMCS_GUEST_FS_BASE:
5630 case VMX_VMCS_GUEST_GS_BASE:
5631 case VMX_VMCS_GUEST_LDTR_BASE:
5632 case VMX_VMCS_GUEST_TR_BASE:
5633 case VMX_VMCS_GUEST_GDTR_BASE:
5634 case VMX_VMCS_GUEST_IDTR_BASE:
5635 case VMX_VMCS_GUEST_RSP:
5636 case VMX_VMCS_GUEST_RIP:
5637 case VMX_VMCS_GUEST_SYSENTER_ESP:
5638 case VMX_VMCS_GUEST_SYSENTER_EIP:
5639 {
5640 if (!(RT_HI_U32(u64Val)))
5641 {
5642 /* If this field is 64-bit, VT-x will zero out the top bits. */
5643 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5644 }
5645 else
5646 {
5647 /* Assert that only the 32->64 switcher case should ever come here. */
5648 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5649 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5650 }
5651 break;
5652 }
5653
5654 default:
5655 {
5656 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5657 rc = VERR_INVALID_PARAMETER;
5658 break;
5659 }
5660 }
5661 AssertRCReturn(rc, rc);
5662 return rc;
5663}
5664
5665
5666/**
5667 * Queue up a VMWRITE by using the VMCS write cache.
5668 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5669 *
5670 * @param pVCpu The cross context virtual CPU structure.
5671 * @param idxField The VMCS field encoding.
5672 * @param u64Val 16, 32 or 64-bit value.
5673 */
5674VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5675{
5676 AssertPtr(pVCpu);
5677 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5678
5679 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5680 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5681
5682 /* Make sure there are no duplicates. */
5683 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5684 {
5685 if (pCache->Write.aField[i] == idxField)
5686 {
5687 pCache->Write.aFieldVal[i] = u64Val;
5688 return VINF_SUCCESS;
5689 }
5690 }
5691
5692 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5693 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5694 pCache->Write.cValidEntries++;
5695 return VINF_SUCCESS;
5696}
5697#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5698
5699
5700/**
5701 * Sets up the usage of TSC-offsetting and updates the VMCS.
5702 *
5703 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5704 * VMX preemption timer.
5705 *
5706 * @returns VBox status code.
5707 * @param pVM The cross context VM structure.
5708 * @param pVCpu The cross context virtual CPU structure.
5709 *
5710 * @remarks No-long-jump zone!!!
5711 */
5712static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5713{
5714 int rc;
5715 bool fOffsettedTsc;
5716 bool fParavirtTsc;
5717 if (pVM->hm.s.vmx.fUsePreemptTimer)
5718 {
5719 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5720 &fOffsettedTsc, &fParavirtTsc);
5721
5722 /* Make sure the returned values have sane upper and lower boundaries. */
5723 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5724 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5725 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5726 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5727
5728 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5729 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5730 }
5731 else
5732 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5733
5734 /** @todo later optimize this to be done elsewhere and not before every
5735 * VM-entry. */
5736 if (fParavirtTsc)
5737 {
5738 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5739 information before every VM-entry, hence disable it for performance sake. */
5740#if 0
5741 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5742 AssertRC(rc);
5743#endif
5744 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5745 }
5746
5747 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5748 {
5749 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5750 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5751
5752 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5753 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5754 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5755 }
5756 else
5757 {
5758 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5759 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5760 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5761 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5762 }
5763}
5764
5765
5766#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5767/**
5768 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5769 * VM-exit interruption info type.
5770 *
5771 * @returns The IEM exception flags.
5772 * @param uVector The event vector.
5773 * @param uVmxVectorType The VMX event type.
5774 *
5775 * @remarks This function currently only constructs flags required for
5776 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5777 * and CR2 aspects of an exception are not included).
5778 */
5779static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5780{
5781 uint32_t fIemXcptFlags;
5782 switch (uVmxVectorType)
5783 {
5784 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5785 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5786 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5787 break;
5788
5789 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5790 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5791 break;
5792
5793 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5794 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5795 break;
5796
5797 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5798 {
5799 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5800 if (uVector == X86_XCPT_BP)
5801 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5802 else if (uVector == X86_XCPT_OF)
5803 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5804 else
5805 {
5806 fIemXcptFlags = 0;
5807 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5808 }
5809 break;
5810 }
5811
5812 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5813 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5814 break;
5815
5816 default:
5817 fIemXcptFlags = 0;
5818 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5819 break;
5820 }
5821 return fIemXcptFlags;
5822}
5823
5824#else
5825/**
5826 * Determines if an exception is a contributory exception.
5827 *
5828 * Contributory exceptions are ones which can cause double-faults unless the
5829 * original exception was a benign exception. Page-fault is intentionally not
5830 * included here as it's a conditional contributory exception.
5831 *
5832 * @returns true if the exception is contributory, false otherwise.
5833 * @param uVector The exception vector.
5834 */
5835DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5836{
5837 switch (uVector)
5838 {
5839 case X86_XCPT_GP:
5840 case X86_XCPT_SS:
5841 case X86_XCPT_NP:
5842 case X86_XCPT_TS:
5843 case X86_XCPT_DE:
5844 return true;
5845 default:
5846 break;
5847 }
5848 return false;
5849}
5850#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5851
5852
5853/**
5854 * Sets an event as a pending event to be injected into the guest.
5855 *
5856 * @param pVCpu The cross context virtual CPU structure.
5857 * @param u32IntInfo The VM-entry interruption-information field.
5858 * @param cbInstr The VM-entry instruction length in bytes (for software
5859 * interrupts, exceptions and privileged software
5860 * exceptions).
5861 * @param u32ErrCode The VM-entry exception error code.
5862 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5863 * page-fault.
5864 *
5865 * @remarks Statistics counter assumes this is a guest event being injected or
5866 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5867 * always incremented.
5868 */
5869DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5870 RTGCUINTPTR GCPtrFaultAddress)
5871{
5872 Assert(!pVCpu->hm.s.Event.fPending);
5873 pVCpu->hm.s.Event.fPending = true;
5874 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5875 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5876 pVCpu->hm.s.Event.cbInstr = cbInstr;
5877 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5878}
5879
5880
5881/**
5882 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5883 *
5884 * @param pVCpu The cross context virtual CPU structure.
5885 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5886 * out-of-sync. Make sure to update the required fields
5887 * before using them.
5888 */
5889DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5890{
5891 NOREF(pMixedCtx);
5892 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5893 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5894 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5895 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5896}
5897
5898
5899/**
5900 * Handle a condition that occurred while delivering an event through the guest
5901 * IDT.
5902 *
5903 * @returns Strict VBox status code (i.e. informational status codes too).
5904 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5905 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5906 * to continue execution of the guest which will delivery the \#DF.
5907 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5908 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5909 *
5910 * @param pVCpu The cross context virtual CPU structure.
5911 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5912 * out-of-sync. Make sure to update the required fields
5913 * before using them.
5914 * @param pVmxTransient Pointer to the VMX transient structure.
5915 *
5916 * @remarks No-long-jump zone!!!
5917 */
5918static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5919{
5920 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5921
5922 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5923 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5924
5925 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5926 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5927 {
5928 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5929 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5930#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5931 /*
5932 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5933 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5934 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5935 * exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5936 * currently has no way of storing that these exceptions were caused by these instructions
5937 * (ICEBP's #DB poses the problem).
5938 */
5939 IEMXCPTRAISE enmRaise;
5940 IEMXCPTRAISEINFO fRaiseInfo;
5941 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5942 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5943 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5944 {
5945 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5946 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5947 }
5948 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5949 {
5950 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5951 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5952 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5953 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5954 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5955 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5956 uExitVectorType), VERR_VMX_IPE_5);
5957 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5958
5959 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5960 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5961 {
5962 pVmxTransient->fVectoringPF = true;
5963 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5964 }
5965 }
5966 else
5967 {
5968 /*
5969 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5970 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5971 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5972 */
5973 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5974 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5975 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5976 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5977 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5978 }
5979
5980 /*
5981 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5982 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5983 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5984 * subsequent VM-entry would fail.
5985 *
5986 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5987 */
5988 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5989 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5990 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5991 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5992 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5993 {
5994 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5995 }
5996
5997 switch (enmRaise)
5998 {
5999 case IEMXCPTRAISE_CURRENT_XCPT:
6000 {
6001 Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
6002 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
6003 Assert(rcStrict == VINF_SUCCESS);
6004 break;
6005 }
6006
6007 case IEMXCPTRAISE_PREV_EVENT:
6008 {
6009 uint32_t u32ErrCode;
6010 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6011 {
6012 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6013 AssertRCReturn(rc2, rc2);
6014 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6015 }
6016 else
6017 u32ErrCode = 0;
6018
6019 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6020 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6021 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6022 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6023
6024 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6025 pVCpu->hm.s.Event.u32ErrCode));
6026 Assert(rcStrict == VINF_SUCCESS);
6027 break;
6028 }
6029
6030 case IEMXCPTRAISE_REEXEC_INSTR:
6031 Assert(rcStrict == VINF_SUCCESS);
6032 break;
6033
6034 case IEMXCPTRAISE_DOUBLE_FAULT:
6035 {
6036 /*
6037 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6038 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6039 */
6040 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6041 {
6042 pVmxTransient->fVectoringDoublePF = true;
6043 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6044 pMixedCtx->cr2));
6045 rcStrict = VINF_SUCCESS;
6046 }
6047 else
6048 {
6049 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6050 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6051 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6052 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6053 rcStrict = VINF_HM_DOUBLE_FAULT;
6054 }
6055 break;
6056 }
6057
6058 case IEMXCPTRAISE_TRIPLE_FAULT:
6059 {
6060 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6061 uExitVector));
6062 rcStrict = VINF_EM_RESET;
6063 break;
6064 }
6065
6066 case IEMXCPTRAISE_CPU_HANG:
6067 {
6068 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6069 rcStrict = VERR_EM_GUEST_CPU_HANG;
6070 break;
6071 }
6072
6073 default:
6074 {
6075 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6076 rcStrict = VERR_VMX_IPE_2;
6077 break;
6078 }
6079 }
6080#else
6081 typedef enum
6082 {
6083 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6084 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6085 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6086 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6087 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6088 } VMXREFLECTXCPT;
6089
6090 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6091 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6092 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6093 {
6094 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6095 {
6096 enmReflect = VMXREFLECTXCPT_XCPT;
6097#ifdef VBOX_STRICT
6098 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6099 && uExitVector == X86_XCPT_PF)
6100 {
6101 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6102 }
6103#endif
6104 if ( uExitVector == X86_XCPT_PF
6105 && uIdtVector == X86_XCPT_PF)
6106 {
6107 pVmxTransient->fVectoringDoublePF = true;
6108 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6109 }
6110 else if ( uExitVector == X86_XCPT_AC
6111 && uIdtVector == X86_XCPT_AC)
6112 {
6113 enmReflect = VMXREFLECTXCPT_HANG;
6114 Log4(("IDT: Nested #AC - Bad guest\n"));
6115 }
6116 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6117 && hmR0VmxIsContributoryXcpt(uExitVector)
6118 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6119 || uIdtVector == X86_XCPT_PF))
6120 {
6121 enmReflect = VMXREFLECTXCPT_DF;
6122 }
6123 else if (uIdtVector == X86_XCPT_DF)
6124 enmReflect = VMXREFLECTXCPT_TF;
6125 }
6126 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6127 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6128 {
6129 /*
6130 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6131 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6132 */
6133 enmReflect = VMXREFLECTXCPT_XCPT;
6134
6135 if (uExitVector == X86_XCPT_PF)
6136 {
6137 pVmxTransient->fVectoringPF = true;
6138 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6139 }
6140 }
6141 }
6142 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6143 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6144 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6145 {
6146 /*
6147 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6148 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6149 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6150 */
6151 enmReflect = VMXREFLECTXCPT_XCPT;
6152 }
6153
6154 /*
6155 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6156 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6157 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6158 *
6159 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6160 */
6161 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6162 && enmReflect == VMXREFLECTXCPT_XCPT
6163 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6164 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6165 {
6166 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6167 }
6168
6169 switch (enmReflect)
6170 {
6171 case VMXREFLECTXCPT_XCPT:
6172 {
6173 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6174 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6175 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6176
6177 uint32_t u32ErrCode = 0;
6178 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6179 {
6180 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6181 AssertRCReturn(rc2, rc2);
6182 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6183 }
6184
6185 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6186 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6187 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6188 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6189 rcStrict = VINF_SUCCESS;
6190 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6191 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6192
6193 break;
6194 }
6195
6196 case VMXREFLECTXCPT_DF:
6197 {
6198 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6199 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6200 rcStrict = VINF_HM_DOUBLE_FAULT;
6201 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6202 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6203
6204 break;
6205 }
6206
6207 case VMXREFLECTXCPT_TF:
6208 {
6209 rcStrict = VINF_EM_RESET;
6210 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6211 uExitVector));
6212 break;
6213 }
6214
6215 case VMXREFLECTXCPT_HANG:
6216 {
6217 rcStrict = VERR_EM_GUEST_CPU_HANG;
6218 break;
6219 }
6220
6221 default:
6222 Assert(rcStrict == VINF_SUCCESS);
6223 break;
6224 }
6225#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6226 }
6227 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6228 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6229 && uExitVector != X86_XCPT_DF
6230 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6231 {
6232 /*
6233 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6234 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6235 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6236 */
6237 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6238 {
6239 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6240 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6241 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6242 }
6243 }
6244
6245 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6246 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6247 return rcStrict;
6248}
6249
6250
6251/**
6252 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6253 *
6254 * @returns VBox status code.
6255 * @param pVCpu The cross context virtual CPU structure.
6256 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6257 * out-of-sync. Make sure to update the required fields
6258 * before using them.
6259 *
6260 * @remarks No-long-jump zone!!!
6261 */
6262static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6263{
6264 NOREF(pMixedCtx);
6265
6266 /*
6267 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6268 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6269 */
6270 VMMRZCallRing3Disable(pVCpu);
6271 HM_DISABLE_PREEMPT();
6272
6273 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6274 {
6275#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6276 * and 'dbgc-init' containing:
6277 * sxe "xcpt_de"
6278 * sxe "xcpt_bp"
6279 * sxi "xcpt_gp"
6280 * sxi "xcpt_ss"
6281 * sxi "xcpt_np"
6282 */
6283 /** @todo r=ramshankar: Should be fixed after r119291. */
6284 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6285#endif
6286 uint32_t uVal = 0;
6287 uint32_t uShadow = 0;
6288 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6289 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6290 AssertRCReturn(rc, rc);
6291
6292 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6293 CPUMSetGuestCR0(pVCpu, uVal);
6294 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6295 }
6296
6297 HM_RESTORE_PREEMPT();
6298 VMMRZCallRing3Enable(pVCpu);
6299 return VINF_SUCCESS;
6300}
6301
6302
6303/**
6304 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6305 *
6306 * @returns VBox status code.
6307 * @param pVCpu The cross context virtual CPU structure.
6308 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6309 * out-of-sync. Make sure to update the required fields
6310 * before using them.
6311 *
6312 * @remarks No-long-jump zone!!!
6313 */
6314static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6315{
6316 NOREF(pMixedCtx);
6317
6318 int rc = VINF_SUCCESS;
6319 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6320 {
6321 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6322 uint32_t uVal = 0;
6323 uint32_t uShadow = 0;
6324 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6325 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6326 AssertRCReturn(rc, rc);
6327
6328 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6329 CPUMSetGuestCR4(pVCpu, uVal);
6330 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6331 }
6332 return rc;
6333}
6334
6335
6336/**
6337 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6338 *
6339 * @returns VBox status code.
6340 * @param pVCpu The cross context virtual CPU structure.
6341 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6342 * out-of-sync. Make sure to update the required fields
6343 * before using them.
6344 *
6345 * @remarks No-long-jump zone!!!
6346 */
6347static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6348{
6349 int rc = VINF_SUCCESS;
6350 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6351 {
6352 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6353 uint64_t u64Val = 0;
6354 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6355 AssertRCReturn(rc, rc);
6356
6357 pMixedCtx->rip = u64Val;
6358 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6359 }
6360 return rc;
6361}
6362
6363
6364/**
6365 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6366 *
6367 * @returns VBox status code.
6368 * @param pVCpu The cross context virtual CPU structure.
6369 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6370 * out-of-sync. Make sure to update the required fields
6371 * before using them.
6372 *
6373 * @remarks No-long-jump zone!!!
6374 */
6375static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6376{
6377 int rc = VINF_SUCCESS;
6378 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6379 {
6380 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6381 uint64_t u64Val = 0;
6382 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6383 AssertRCReturn(rc, rc);
6384
6385 pMixedCtx->rsp = u64Val;
6386 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6387 }
6388 return rc;
6389}
6390
6391
6392/**
6393 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6394 *
6395 * @returns VBox status code.
6396 * @param pVCpu The cross context virtual CPU structure.
6397 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6398 * out-of-sync. Make sure to update the required fields
6399 * before using them.
6400 *
6401 * @remarks No-long-jump zone!!!
6402 */
6403static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6404{
6405 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6406 {
6407 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6408 uint32_t uVal = 0;
6409 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6410 AssertRCReturn(rc, rc);
6411
6412 pMixedCtx->eflags.u32 = uVal;
6413 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6414 {
6415 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6416 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6417
6418 pMixedCtx->eflags.Bits.u1VM = 0;
6419 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6420 }
6421
6422 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6423 }
6424 return VINF_SUCCESS;
6425}
6426
6427
6428/**
6429 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6430 * guest-CPU context.
6431 */
6432DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6433{
6434 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6435 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6436 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6437 return rc;
6438}
6439
6440
6441/**
6442 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6443 * from the guest-state area in the VMCS.
6444 *
6445 * @param pVCpu The cross context virtual CPU structure.
6446 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6447 * out-of-sync. Make sure to update the required fields
6448 * before using them.
6449 *
6450 * @remarks No-long-jump zone!!!
6451 */
6452static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6453{
6454 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6455 {
6456 uint32_t uIntrState = 0;
6457 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6458 AssertRC(rc);
6459
6460 if (!uIntrState)
6461 {
6462 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6463 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6464
6465 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6466 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6467 }
6468 else
6469 {
6470 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6471 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6472 {
6473 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6474 AssertRC(rc);
6475 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6476 AssertRC(rc);
6477
6478 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6479 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6480 }
6481 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6482 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6483
6484 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6485 {
6486 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6487 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6488 }
6489 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6490 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6491 }
6492
6493 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6494 }
6495}
6496
6497
6498/**
6499 * Saves the guest's activity state.
6500 *
6501 * @returns VBox status code.
6502 * @param pVCpu The cross context virtual CPU structure.
6503 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6504 * out-of-sync. Make sure to update the required fields
6505 * before using them.
6506 *
6507 * @remarks No-long-jump zone!!!
6508 */
6509static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6510{
6511 NOREF(pMixedCtx);
6512 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6513 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6514 return VINF_SUCCESS;
6515}
6516
6517
6518/**
6519 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6520 * the current VMCS into the guest-CPU context.
6521 *
6522 * @returns VBox status code.
6523 * @param pVCpu The cross context virtual CPU structure.
6524 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6525 * out-of-sync. Make sure to update the required fields
6526 * before using them.
6527 *
6528 * @remarks No-long-jump zone!!!
6529 */
6530static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6531{
6532 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6533 {
6534 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6535 uint32_t u32Val = 0;
6536 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6537 pMixedCtx->SysEnter.cs = u32Val;
6538 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6539 }
6540
6541 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6542 {
6543 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6544 uint64_t u64Val = 0;
6545 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6546 pMixedCtx->SysEnter.eip = u64Val;
6547 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6548 }
6549 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6550 {
6551 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6552 uint64_t u64Val = 0;
6553 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6554 pMixedCtx->SysEnter.esp = u64Val;
6555 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6556 }
6557 return VINF_SUCCESS;
6558}
6559
6560
6561/**
6562 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6563 * the CPU back into the guest-CPU context.
6564 *
6565 * @returns VBox status code.
6566 * @param pVCpu The cross context virtual CPU structure.
6567 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6568 * out-of-sync. Make sure to update the required fields
6569 * before using them.
6570 *
6571 * @remarks No-long-jump zone!!!
6572 */
6573static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6574{
6575 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6576 VMMRZCallRing3Disable(pVCpu);
6577 HM_DISABLE_PREEMPT();
6578
6579 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6580 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6581 {
6582 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6583 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6584 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6585 }
6586
6587 HM_RESTORE_PREEMPT();
6588 VMMRZCallRing3Enable(pVCpu);
6589
6590 return VINF_SUCCESS;
6591}
6592
6593
6594/**
6595 * Saves the auto load/store'd guest MSRs from the current VMCS into
6596 * the guest-CPU context.
6597 *
6598 * @returns VBox status code.
6599 * @param pVCpu The cross context virtual CPU structure.
6600 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6601 * out-of-sync. Make sure to update the required fields
6602 * before using them.
6603 *
6604 * @remarks No-long-jump zone!!!
6605 */
6606static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6607{
6608 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6609 return VINF_SUCCESS;
6610
6611 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6612 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6613 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6614 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6615 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6616 {
6617 switch (pMsr->u32Msr)
6618 {
6619 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6620 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6621 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6622 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6623 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6624 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6625 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6626 break;
6627
6628 default:
6629 {
6630 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6631 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6632 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6633 }
6634 }
6635 }
6636
6637 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6638 return VINF_SUCCESS;
6639}
6640
6641
6642/**
6643 * Saves the guest control registers from the current VMCS into the guest-CPU
6644 * context.
6645 *
6646 * @returns VBox status code.
6647 * @param pVCpu The cross context virtual CPU structure.
6648 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6649 * out-of-sync. Make sure to update the required fields
6650 * before using them.
6651 *
6652 * @remarks No-long-jump zone!!!
6653 */
6654static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6655{
6656 /* Guest CR0. Guest FPU. */
6657 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6658 AssertRCReturn(rc, rc);
6659
6660 /* Guest CR4. */
6661 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6662 AssertRCReturn(rc, rc);
6663
6664 /* Guest CR2 - updated always during the world-switch or in #PF. */
6665 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6666 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6667 {
6668 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6669 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6670 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6671
6672 PVM pVM = pVCpu->CTX_SUFF(pVM);
6673 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6674 || ( pVM->hm.s.fNestedPaging
6675 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6676 {
6677 uint64_t u64Val = 0;
6678 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6679 if (pMixedCtx->cr3 != u64Val)
6680 {
6681 CPUMSetGuestCR3(pVCpu, u64Val);
6682 if (VMMRZCallRing3IsEnabled(pVCpu))
6683 {
6684 PGMUpdateCR3(pVCpu, u64Val);
6685 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6686 }
6687 else
6688 {
6689 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6690 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6691 }
6692 }
6693
6694 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6695 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6696 {
6697 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6698 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6699 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6700 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6701 AssertRCReturn(rc, rc);
6702
6703 if (VMMRZCallRing3IsEnabled(pVCpu))
6704 {
6705 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6706 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6707 }
6708 else
6709 {
6710 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6711 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6712 }
6713 }
6714 }
6715
6716 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6717 }
6718
6719 /*
6720 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6721 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6722 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6723 *
6724 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6725 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6726 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6727 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6728 *
6729 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6730 */
6731 if (VMMRZCallRing3IsEnabled(pVCpu))
6732 {
6733 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6734 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6735
6736 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6737 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6738
6739 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6740 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6741 }
6742
6743 return rc;
6744}
6745
6746
6747/**
6748 * Reads a guest segment register from the current VMCS into the guest-CPU
6749 * context.
6750 *
6751 * @returns VBox status code.
6752 * @param pVCpu The cross context virtual CPU structure.
6753 * @param idxSel Index of the selector in the VMCS.
6754 * @param idxLimit Index of the segment limit in the VMCS.
6755 * @param idxBase Index of the segment base in the VMCS.
6756 * @param idxAccess Index of the access rights of the segment in the VMCS.
6757 * @param pSelReg Pointer to the segment selector.
6758 *
6759 * @remarks No-long-jump zone!!!
6760 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6761 * macro as that takes care of whether to read from the VMCS cache or
6762 * not.
6763 */
6764DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6765 PCPUMSELREG pSelReg)
6766{
6767 NOREF(pVCpu);
6768
6769 uint32_t u32Val = 0;
6770 int rc = VMXReadVmcs32(idxSel, &u32Val);
6771 AssertRCReturn(rc, rc);
6772 pSelReg->Sel = (uint16_t)u32Val;
6773 pSelReg->ValidSel = (uint16_t)u32Val;
6774 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6775
6776 rc = VMXReadVmcs32(idxLimit, &u32Val);
6777 AssertRCReturn(rc, rc);
6778 pSelReg->u32Limit = u32Val;
6779
6780 uint64_t u64Val = 0;
6781 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6782 AssertRCReturn(rc, rc);
6783 pSelReg->u64Base = u64Val;
6784
6785 rc = VMXReadVmcs32(idxAccess, &u32Val);
6786 AssertRCReturn(rc, rc);
6787 pSelReg->Attr.u = u32Val;
6788
6789 /*
6790 * If VT-x marks the segment as unusable, most other bits remain undefined:
6791 * - For CS the L, D and G bits have meaning.
6792 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6793 * - For the remaining data segments no bits are defined.
6794 *
6795 * The present bit and the unusable bit has been observed to be set at the
6796 * same time (the selector was supposed to be invalid as we started executing
6797 * a V8086 interrupt in ring-0).
6798 *
6799 * What should be important for the rest of the VBox code, is that the P bit is
6800 * cleared. Some of the other VBox code recognizes the unusable bit, but
6801 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6802 * safe side here, we'll strip off P and other bits we don't care about. If
6803 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6804 *
6805 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6806 */
6807 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6808 {
6809 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6810
6811 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6812 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6813 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6814
6815 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6816#ifdef DEBUG_bird
6817 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6818 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6819 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6820#endif
6821 }
6822 return VINF_SUCCESS;
6823}
6824
6825
6826#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6827# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6828 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6829 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6830#else
6831# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6832 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6833 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6834#endif
6835
6836
6837/**
6838 * Saves the guest segment registers from the current VMCS into the guest-CPU
6839 * context.
6840 *
6841 * @returns VBox status code.
6842 * @param pVCpu The cross context virtual CPU structure.
6843 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6844 * out-of-sync. Make sure to update the required fields
6845 * before using them.
6846 *
6847 * @remarks No-long-jump zone!!!
6848 */
6849static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6850{
6851 /* Guest segment registers. */
6852 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6853 {
6854 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6855 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6856 AssertRCReturn(rc, rc);
6857
6858 rc = VMXLOCAL_READ_SEG(CS, cs);
6859 rc |= VMXLOCAL_READ_SEG(SS, ss);
6860 rc |= VMXLOCAL_READ_SEG(DS, ds);
6861 rc |= VMXLOCAL_READ_SEG(ES, es);
6862 rc |= VMXLOCAL_READ_SEG(FS, fs);
6863 rc |= VMXLOCAL_READ_SEG(GS, gs);
6864 AssertRCReturn(rc, rc);
6865
6866 /* Restore segment attributes for real-on-v86 mode hack. */
6867 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6868 {
6869 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6870 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6871 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6872 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6873 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6874 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6875 }
6876 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6877 }
6878
6879 return VINF_SUCCESS;
6880}
6881
6882
6883/**
6884 * Saves the guest descriptor table registers and task register from the current
6885 * VMCS into the guest-CPU context.
6886 *
6887 * @returns VBox status code.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6890 * out-of-sync. Make sure to update the required fields
6891 * before using them.
6892 *
6893 * @remarks No-long-jump zone!!!
6894 */
6895static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6896{
6897 int rc = VINF_SUCCESS;
6898
6899 /* Guest LDTR. */
6900 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6901 {
6902 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6903 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6904 AssertRCReturn(rc, rc);
6905 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6906 }
6907
6908 /* Guest GDTR. */
6909 uint64_t u64Val = 0;
6910 uint32_t u32Val = 0;
6911 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6912 {
6913 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6914 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6915 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6916 pMixedCtx->gdtr.pGdt = u64Val;
6917 pMixedCtx->gdtr.cbGdt = u32Val;
6918 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6919 }
6920
6921 /* Guest IDTR. */
6922 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6923 {
6924 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6925 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6926 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6927 pMixedCtx->idtr.pIdt = u64Val;
6928 pMixedCtx->idtr.cbIdt = u32Val;
6929 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6930 }
6931
6932 /* Guest TR. */
6933 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6934 {
6935 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6936 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6937 AssertRCReturn(rc, rc);
6938
6939 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6940 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6941 {
6942 rc = VMXLOCAL_READ_SEG(TR, tr);
6943 AssertRCReturn(rc, rc);
6944 }
6945 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6946 }
6947 return rc;
6948}
6949
6950#undef VMXLOCAL_READ_SEG
6951
6952
6953/**
6954 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6955 * context.
6956 *
6957 * @returns VBox status code.
6958 * @param pVCpu The cross context virtual CPU structure.
6959 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6960 * out-of-sync. Make sure to update the required fields
6961 * before using them.
6962 *
6963 * @remarks No-long-jump zone!!!
6964 */
6965static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6966{
6967 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6968 {
6969 if (!pVCpu->hm.s.fUsingHyperDR7)
6970 {
6971 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6972 uint32_t u32Val;
6973 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6974 pMixedCtx->dr[7] = u32Val;
6975 }
6976
6977 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6978 }
6979 return VINF_SUCCESS;
6980}
6981
6982
6983/**
6984 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6985 *
6986 * @returns VBox status code.
6987 * @param pVCpu The cross context virtual CPU structure.
6988 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6989 * out-of-sync. Make sure to update the required fields
6990 * before using them.
6991 *
6992 * @remarks No-long-jump zone!!!
6993 */
6994static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6995{
6996 NOREF(pMixedCtx);
6997
6998 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6999 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
7000 return VINF_SUCCESS;
7001}
7002
7003
7004/**
7005 * Saves the entire guest state from the currently active VMCS into the
7006 * guest-CPU context.
7007 *
7008 * This essentially VMREADs all guest-data.
7009 *
7010 * @returns VBox status code.
7011 * @param pVCpu The cross context virtual CPU structure.
7012 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7013 * out-of-sync. Make sure to update the required fields
7014 * before using them.
7015 */
7016static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7017{
7018 Assert(pVCpu);
7019 Assert(pMixedCtx);
7020
7021 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7022 return VINF_SUCCESS;
7023
7024 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7025 again on the ring-3 callback path, there is no real need to. */
7026 if (VMMRZCallRing3IsEnabled(pVCpu))
7027 VMMR0LogFlushDisable(pVCpu);
7028 else
7029 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7030 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7031
7032 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7033 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7034
7035 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7036 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7037
7038 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7039 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7040
7041 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7042 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7043
7044 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7045 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7046
7047 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7057 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7058
7059 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7060 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7061
7062 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7063 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7064 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7065
7066 if (VMMRZCallRing3IsEnabled(pVCpu))
7067 VMMR0LogFlushEnable(pVCpu);
7068
7069 return VINF_SUCCESS;
7070}
7071
7072
7073/**
7074 * Saves basic guest registers needed for IEM instruction execution.
7075 *
7076 * @returns VBox status code (OR-able).
7077 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7078 * @param pMixedCtx Pointer to the CPU context of the guest.
7079 * @param fMemory Whether the instruction being executed operates on
7080 * memory or not. Only CR0 is synced up if clear.
7081 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7082 */
7083static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7084{
7085 /*
7086 * We assume all general purpose registers other than RSP are available.
7087 *
7088 * - RIP is a must, as it will be incremented or otherwise changed.
7089 * - RFLAGS are always required to figure the CPL.
7090 * - RSP isn't always required, however it's a GPR, so frequently required.
7091 * - SS and CS are the only segment register needed if IEM doesn't do memory
7092 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7093 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7094 * be required for memory accesses.
7095 *
7096 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7097 */
7098 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7099 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7100 if (fNeedRsp)
7101 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7102 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /** @todo Only CS and SS are required here. */
7103 if (!fMemory)
7104 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7105 else
7106 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7107 AssertRCReturn(rc, rc);
7108 return rc;
7109}
7110
7111
7112/**
7113 * Ensures that we've got a complete basic guest-context.
7114 *
7115 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7116 * is for the interpreter.
7117 *
7118 * @returns VBox status code.
7119 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7120 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7121 * needing to be synced in.
7122 * @thread EMT(pVCpu)
7123 */
7124VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7125{
7126 /* Note! Since this is only applicable to VT-x, the implementation is placed
7127 in the VT-x part of the sources instead of the generic stuff. */
7128 int rc;
7129 PVM pVM = pVCpu->CTX_SUFF(pVM);
7130 if ( pVM->hm.s.vmx.fSupported
7131 && VM_IS_HM_ENABLED(pVM))
7132 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7133 else
7134 rc = VINF_SUCCESS;
7135
7136 /*
7137 * For now, imply that the caller might change everything too. Do this after
7138 * saving the guest state so as to not trigger assertions.
7139 *
7140 * This is required for AMD-V too as it too only selectively re-loads changed
7141 * guest state back in to the VMCB.
7142 */
7143 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7144 return rc;
7145}
7146
7147
7148/**
7149 * Check per-VM and per-VCPU force flag actions that require us to go back to
7150 * ring-3 for one reason or another.
7151 *
7152 * @returns Strict VBox status code (i.e. informational status codes too)
7153 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7154 * ring-3.
7155 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7156 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7157 * interrupts)
7158 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7159 * all EMTs to be in ring-3.
7160 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7161 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7162 * to the EM loop.
7163 *
7164 * @param pVM The cross context VM structure.
7165 * @param pVCpu The cross context virtual CPU structure.
7166 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7167 * out-of-sync. Make sure to update the required fields
7168 * before using them.
7169 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7170 */
7171static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7172{
7173 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7174
7175 /*
7176 * Anything pending? Should be more likely than not if we're doing a good job.
7177 */
7178 if ( !fStepping
7179 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7180 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7181 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7182 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7183 return VINF_SUCCESS;
7184
7185 /* We need the control registers now, make sure the guest-CPU context is updated. */
7186 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7187 AssertRCReturn(rc3, rc3);
7188
7189 /* Pending HM CR3 sync. */
7190 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7191 {
7192 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7193 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7194 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7195 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7196 }
7197
7198 /* Pending HM PAE PDPEs. */
7199 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7200 {
7201 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7202 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7203 }
7204
7205 /* Pending PGM C3 sync. */
7206 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7207 {
7208 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7209 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7210 if (rcStrict2 != VINF_SUCCESS)
7211 {
7212 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7213 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7214 return rcStrict2;
7215 }
7216 }
7217
7218 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7219 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7220 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7221 {
7222 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7223 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7224 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7225 return rc2;
7226 }
7227
7228 /* Pending VM request packets, such as hardware interrupts. */
7229 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7230 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7231 {
7232 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7233 return VINF_EM_PENDING_REQUEST;
7234 }
7235
7236 /* Pending PGM pool flushes. */
7237 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7238 {
7239 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7240 return VINF_PGM_POOL_FLUSH_PENDING;
7241 }
7242
7243 /* Pending DMA requests. */
7244 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7245 {
7246 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7247 return VINF_EM_RAW_TO_R3;
7248 }
7249
7250 return VINF_SUCCESS;
7251}
7252
7253
7254/**
7255 * Converts any TRPM trap into a pending HM event. This is typically used when
7256 * entering from ring-3 (not longjmp returns).
7257 *
7258 * @param pVCpu The cross context virtual CPU structure.
7259 */
7260static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7261{
7262 Assert(TRPMHasTrap(pVCpu));
7263 Assert(!pVCpu->hm.s.Event.fPending);
7264
7265 uint8_t uVector;
7266 TRPMEVENT enmTrpmEvent;
7267 RTGCUINT uErrCode;
7268 RTGCUINTPTR GCPtrFaultAddress;
7269 uint8_t cbInstr;
7270
7271 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7272 AssertRC(rc);
7273
7274 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7275 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7276 if (enmTrpmEvent == TRPM_TRAP)
7277 {
7278 switch (uVector)
7279 {
7280 case X86_XCPT_NMI:
7281 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7282 break;
7283
7284 case X86_XCPT_BP:
7285 case X86_XCPT_OF:
7286 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7287 break;
7288
7289 case X86_XCPT_PF:
7290 case X86_XCPT_DF:
7291 case X86_XCPT_TS:
7292 case X86_XCPT_NP:
7293 case X86_XCPT_SS:
7294 case X86_XCPT_GP:
7295 case X86_XCPT_AC:
7296 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7297 RT_FALL_THRU();
7298 default:
7299 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7300 break;
7301 }
7302 }
7303 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7304 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7305 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7306 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7307 else
7308 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7309
7310 rc = TRPMResetTrap(pVCpu);
7311 AssertRC(rc);
7312 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7313 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7314
7315 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7316}
7317
7318
7319/**
7320 * Converts the pending HM event into a TRPM trap.
7321 *
7322 * @param pVCpu The cross context virtual CPU structure.
7323 */
7324static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7325{
7326 Assert(pVCpu->hm.s.Event.fPending);
7327
7328 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7329 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7330 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7331 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7332
7333 /* If a trap was already pending, we did something wrong! */
7334 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7335
7336 TRPMEVENT enmTrapType;
7337 switch (uVectorType)
7338 {
7339 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7340 enmTrapType = TRPM_HARDWARE_INT;
7341 break;
7342
7343 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7344 enmTrapType = TRPM_SOFTWARE_INT;
7345 break;
7346
7347 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7348 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7349 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7350 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7351 enmTrapType = TRPM_TRAP;
7352 break;
7353
7354 default:
7355 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7356 enmTrapType = TRPM_32BIT_HACK;
7357 break;
7358 }
7359
7360 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7361
7362 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7363 AssertRC(rc);
7364
7365 if (fErrorCodeValid)
7366 TRPMSetErrorCode(pVCpu, uErrorCode);
7367
7368 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7369 && uVector == X86_XCPT_PF)
7370 {
7371 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7372 }
7373 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7374 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7375 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7376 {
7377 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7378 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7379 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7380 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7381 }
7382
7383 /* Clear any pending events from the VMCS. */
7384 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7385 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7386
7387 /* We're now done converting the pending event. */
7388 pVCpu->hm.s.Event.fPending = false;
7389}
7390
7391
7392/**
7393 * Does the necessary state syncing before returning to ring-3 for any reason
7394 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7395 *
7396 * @returns VBox status code.
7397 * @param pVCpu The cross context virtual CPU structure.
7398 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7399 * be out-of-sync. Make sure to update the required
7400 * fields before using them.
7401 * @param fSaveGuestState Whether to save the guest state or not.
7402 *
7403 * @remarks No-long-jmp zone!!!
7404 */
7405static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7406{
7407 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7408 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7409
7410 RTCPUID idCpu = RTMpCpuId();
7411 Log4Func(("HostCpuId=%u\n", idCpu));
7412
7413 /*
7414 * !!! IMPORTANT !!!
7415 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7416 */
7417
7418 /* Save the guest state if necessary. */
7419 if ( fSaveGuestState
7420 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7421 {
7422 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7423 AssertRCReturn(rc, rc);
7424 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7425 }
7426
7427 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7428 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7429 {
7430 /* We shouldn't reload CR0 without saving it first. */
7431 if (!fSaveGuestState)
7432 {
7433 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7434 AssertRCReturn(rc, rc);
7435 }
7436 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7437 }
7438
7439 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7440#ifdef VBOX_STRICT
7441 if (CPUMIsHyperDebugStateActive(pVCpu))
7442 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7443#endif
7444 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7445 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7446 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7447 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7448
7449#if HC_ARCH_BITS == 64
7450 /* Restore host-state bits that VT-x only restores partially. */
7451 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7452 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7453 {
7454 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7455 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7456 }
7457 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7458#endif
7459
7460 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7461 if (pVCpu->hm.s.vmx.fLazyMsrs)
7462 {
7463 /* We shouldn't reload the guest MSRs without saving it first. */
7464 if (!fSaveGuestState)
7465 {
7466 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7467 AssertRCReturn(rc, rc);
7468 }
7469 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7470 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7471 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7472 }
7473
7474 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7475 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7476
7477 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7478 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7479 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7480 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7481 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7482 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7483 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7484 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7485
7486 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7487
7488 /** @todo This partially defeats the purpose of having preemption hooks.
7489 * The problem is, deregistering the hooks should be moved to a place that
7490 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7491 * context.
7492 */
7493 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7494 {
7495 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7496 AssertRCReturn(rc, rc);
7497
7498 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7499 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7500 }
7501 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7502 NOREF(idCpu);
7503
7504 return VINF_SUCCESS;
7505}
7506
7507
7508/**
7509 * Leaves the VT-x session.
7510 *
7511 * @returns VBox status code.
7512 * @param pVCpu The cross context virtual CPU structure.
7513 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7514 * out-of-sync. Make sure to update the required fields
7515 * before using them.
7516 *
7517 * @remarks No-long-jmp zone!!!
7518 */
7519DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7520{
7521 HM_DISABLE_PREEMPT();
7522 HMVMX_ASSERT_CPU_SAFE();
7523 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7524 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7525
7526 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7527 and done this from the VMXR0ThreadCtxCallback(). */
7528 if (!pVCpu->hm.s.fLeaveDone)
7529 {
7530 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7531 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7532 pVCpu->hm.s.fLeaveDone = true;
7533 }
7534 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7535
7536 /*
7537 * !!! IMPORTANT !!!
7538 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7539 */
7540
7541 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7542 /** @todo Deregistering here means we need to VMCLEAR always
7543 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7544 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7545 VMMR0ThreadCtxHookDisable(pVCpu);
7546
7547 /* Leave HM context. This takes care of local init (term). */
7548 int rc = HMR0LeaveCpu(pVCpu);
7549
7550 HM_RESTORE_PREEMPT();
7551 return rc;
7552}
7553
7554
7555/**
7556 * Does the necessary state syncing before doing a longjmp to ring-3.
7557 *
7558 * @returns VBox status code.
7559 * @param pVCpu The cross context virtual CPU structure.
7560 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7561 * out-of-sync. Make sure to update the required fields
7562 * before using them.
7563 *
7564 * @remarks No-long-jmp zone!!!
7565 */
7566DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7567{
7568 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7569}
7570
7571
7572/**
7573 * Take necessary actions before going back to ring-3.
7574 *
7575 * An action requires us to go back to ring-3. This function does the necessary
7576 * steps before we can safely return to ring-3. This is not the same as longjmps
7577 * to ring-3, this is voluntary and prepares the guest so it may continue
7578 * executing outside HM (recompiler/IEM).
7579 *
7580 * @returns VBox status code.
7581 * @param pVM The cross context VM structure.
7582 * @param pVCpu The cross context virtual CPU structure.
7583 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7584 * out-of-sync. Make sure to update the required fields
7585 * before using them.
7586 * @param rcExit The reason for exiting to ring-3. Can be
7587 * VINF_VMM_UNKNOWN_RING3_CALL.
7588 */
7589static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7590{
7591 Assert(pVM);
7592 Assert(pVCpu);
7593 Assert(pMixedCtx);
7594 HMVMX_ASSERT_PREEMPT_SAFE();
7595
7596 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7597 {
7598 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7599 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7600 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7601 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7602 }
7603
7604 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7605 VMMRZCallRing3Disable(pVCpu);
7606 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7607
7608 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7609 if (pVCpu->hm.s.Event.fPending)
7610 {
7611 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7612 Assert(!pVCpu->hm.s.Event.fPending);
7613 }
7614
7615 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7616 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7617
7618 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7619 and if we're injecting an event we should have a TRPM trap pending. */
7620 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7621#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7622 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7623#endif
7624
7625 /* Save guest state and restore host state bits. */
7626 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7627 AssertRCReturn(rc, rc);
7628 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7629 /* Thread-context hooks are unregistered at this point!!! */
7630
7631 /* Sync recompiler state. */
7632 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7633 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7634 | CPUM_CHANGED_LDTR
7635 | CPUM_CHANGED_GDTR
7636 | CPUM_CHANGED_IDTR
7637 | CPUM_CHANGED_TR
7638 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7639 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7640 if ( pVM->hm.s.fNestedPaging
7641 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7642 {
7643 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7644 }
7645
7646 Assert(!pVCpu->hm.s.fClearTrapFlag);
7647
7648 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7649 if (rcExit != VINF_EM_RAW_INTERRUPT)
7650 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7651
7652 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7653
7654 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7655 VMMRZCallRing3RemoveNotification(pVCpu);
7656 VMMRZCallRing3Enable(pVCpu);
7657
7658 return rc;
7659}
7660
7661
7662/**
7663 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7664 * longjump to ring-3 and possibly get preempted.
7665 *
7666 * @returns VBox status code.
7667 * @param pVCpu The cross context virtual CPU structure.
7668 * @param enmOperation The operation causing the ring-3 longjump.
7669 * @param pvUser Opaque pointer to the guest-CPU context. The data
7670 * may be out-of-sync. Make sure to update the required
7671 * fields before using them.
7672 */
7673static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7674{
7675 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7676 {
7677 /*
7678 * !!! IMPORTANT !!!
7679 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7680 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7681 */
7682 VMMRZCallRing3RemoveNotification(pVCpu);
7683 VMMRZCallRing3Disable(pVCpu);
7684 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7685 RTThreadPreemptDisable(&PreemptState);
7686
7687 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7688 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7689
7690#if HC_ARCH_BITS == 64
7691 /* Restore host-state bits that VT-x only restores partially. */
7692 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7693 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7694 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7695 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7696#endif
7697 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7698 if (pVCpu->hm.s.vmx.fLazyMsrs)
7699 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7700
7701 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7702 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7703 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7704 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7705 {
7706 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7707 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7708 }
7709
7710 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7711 VMMR0ThreadCtxHookDisable(pVCpu);
7712 HMR0LeaveCpu(pVCpu);
7713 RTThreadPreemptRestore(&PreemptState);
7714 return VINF_SUCCESS;
7715 }
7716
7717 Assert(pVCpu);
7718 Assert(pvUser);
7719 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7720 HMVMX_ASSERT_PREEMPT_SAFE();
7721
7722 VMMRZCallRing3Disable(pVCpu);
7723 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7724
7725 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7726 enmOperation));
7727
7728 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7729 AssertRCReturn(rc, rc);
7730
7731 VMMRZCallRing3Enable(pVCpu);
7732 return VINF_SUCCESS;
7733}
7734
7735
7736/**
7737 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7738 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7739 *
7740 * @param pVCpu The cross context virtual CPU structure.
7741 */
7742DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7743{
7744 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7745 {
7746 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7747 {
7748 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7749 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7750 AssertRC(rc);
7751 Log4(("Setup interrupt-window exiting\n"));
7752 }
7753 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7754}
7755
7756
7757/**
7758 * Clears the interrupt-window exiting control in the VMCS.
7759 *
7760 * @param pVCpu The cross context virtual CPU structure.
7761 */
7762DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7763{
7764 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7765 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7766 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7767 AssertRC(rc);
7768 Log4(("Cleared interrupt-window exiting\n"));
7769}
7770
7771
7772/**
7773 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7774 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7775 *
7776 * @param pVCpu The cross context virtual CPU structure.
7777 */
7778DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7779{
7780 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7781 {
7782 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7783 {
7784 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7785 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7786 AssertRC(rc);
7787 Log4(("Setup NMI-window exiting\n"));
7788 }
7789 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7790}
7791
7792
7793/**
7794 * Clears the NMI-window exiting control in the VMCS.
7795 *
7796 * @param pVCpu The cross context virtual CPU structure.
7797 */
7798DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7799{
7800 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7801 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7802 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7803 AssertRC(rc);
7804 Log4(("Cleared NMI-window exiting\n"));
7805}
7806
7807
7808/**
7809 * Evaluates the event to be delivered to the guest and sets it as the pending
7810 * event.
7811 *
7812 * @returns The VT-x guest-interruptibility state.
7813 * @param pVCpu The cross context virtual CPU structure.
7814 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7815 * out-of-sync. Make sure to update the required fields
7816 * before using them.
7817 */
7818static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7819{
7820 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7821 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7822 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7823 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7824 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7825
7826 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7827 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7828 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7829 Assert(!TRPMHasTrap(pVCpu));
7830
7831 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7832 APICUpdatePendingInterrupts(pVCpu);
7833
7834 /*
7835 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7836 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7837 */
7838 /** @todo SMI. SMIs take priority over NMIs. */
7839 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7840 {
7841 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7842 if ( !pVCpu->hm.s.Event.fPending
7843 && !fBlockNmi
7844 && !fBlockSti
7845 && !fBlockMovSS)
7846 {
7847 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7848 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7849 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7850
7851 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7852 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7853 }
7854 else
7855 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7856 }
7857 /*
7858 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7859 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7860 */
7861 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7862 && !pVCpu->hm.s.fSingleInstruction)
7863 {
7864 Assert(!DBGFIsStepping(pVCpu));
7865 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7866 AssertRC(rc);
7867 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7868 if ( !pVCpu->hm.s.Event.fPending
7869 && !fBlockInt
7870 && !fBlockSti
7871 && !fBlockMovSS)
7872 {
7873 uint8_t u8Interrupt;
7874 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7875 if (RT_SUCCESS(rc))
7876 {
7877 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7878 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7879 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7880
7881 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7882 }
7883 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7884 {
7885 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7886 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7887 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7888
7889 /*
7890 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7891 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7892 * need to re-set this force-flag here.
7893 */
7894 }
7895 else
7896 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7897 }
7898 else
7899 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7900 }
7901
7902 return uIntrState;
7903}
7904
7905
7906/**
7907 * Sets a pending-debug exception to be delivered to the guest if the guest is
7908 * single-stepping in the VMCS.
7909 *
7910 * @param pVCpu The cross context virtual CPU structure.
7911 */
7912DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7913{
7914 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7915 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7916 AssertRC(rc);
7917}
7918
7919
7920/**
7921 * Injects any pending events into the guest if the guest is in a state to
7922 * receive them.
7923 *
7924 * @returns Strict VBox status code (i.e. informational status codes too).
7925 * @param pVCpu The cross context virtual CPU structure.
7926 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7927 * out-of-sync. Make sure to update the required fields
7928 * before using them.
7929 * @param uIntrState The VT-x guest-interruptibility state.
7930 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7931 * return VINF_EM_DBG_STEPPED if the event was
7932 * dispatched directly.
7933 */
7934static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7935{
7936 HMVMX_ASSERT_PREEMPT_SAFE();
7937 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7938
7939 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7940 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7941
7942 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7943 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7944 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7945 Assert(!TRPMHasTrap(pVCpu));
7946
7947 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7948 if (pVCpu->hm.s.Event.fPending)
7949 {
7950 /*
7951 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7952 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7953 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7954 *
7955 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7956 */
7957 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7958#ifdef VBOX_STRICT
7959 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7960 {
7961 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7962 Assert(!fBlockInt);
7963 Assert(!fBlockSti);
7964 Assert(!fBlockMovSS);
7965 }
7966 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7967 {
7968 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7969 Assert(!fBlockSti);
7970 Assert(!fBlockMovSS);
7971 Assert(!fBlockNmi);
7972 }
7973#endif
7974 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7975 (uint8_t)uIntType));
7976 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7977 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7978 fStepping, &uIntrState);
7979 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7980
7981 /* Update the interruptibility-state as it could have been changed by
7982 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7983 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7984 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7985
7986 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7987 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7988 else
7989 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7990 }
7991
7992 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7993 if ( fBlockSti
7994 || fBlockMovSS)
7995 {
7996 if (!pVCpu->hm.s.fSingleInstruction)
7997 {
7998 /*
7999 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
8000 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
8001 * See Intel spec. 27.3.4 "Saving Non-Register State".
8002 */
8003 Assert(!DBGFIsStepping(pVCpu));
8004 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
8005 AssertRCReturn(rc2, rc2);
8006 if (pMixedCtx->eflags.Bits.u1TF)
8007 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8008 }
8009 else if (pMixedCtx->eflags.Bits.u1TF)
8010 {
8011 /*
8012 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8013 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8014 */
8015 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8016 uIntrState = 0;
8017 }
8018 }
8019
8020 /*
8021 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8022 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8023 */
8024 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8025 AssertRC(rc2);
8026
8027 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8028 NOREF(fBlockMovSS); NOREF(fBlockSti);
8029 return rcStrict;
8030}
8031
8032
8033/**
8034 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8035 *
8036 * @param pVCpu The cross context virtual CPU structure.
8037 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8038 * out-of-sync. Make sure to update the required fields
8039 * before using them.
8040 */
8041DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8042{
8043 NOREF(pMixedCtx);
8044 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8045 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8046}
8047
8048
8049/**
8050 * Injects a double-fault (\#DF) exception into the VM.
8051 *
8052 * @returns Strict VBox status code (i.e. informational status codes too).
8053 * @param pVCpu The cross context virtual CPU structure.
8054 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8055 * out-of-sync. Make sure to update the required fields
8056 * before using them.
8057 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8058 * and should return VINF_EM_DBG_STEPPED if the event
8059 * is injected directly (register modified by us, not
8060 * by hardware on VM-entry).
8061 * @param puIntrState Pointer to the current guest interruptibility-state.
8062 * This interruptibility-state will be updated if
8063 * necessary. This cannot not be NULL.
8064 */
8065DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8066{
8067 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8068 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8069 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8070 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8071 fStepping, puIntrState);
8072}
8073
8074
8075/**
8076 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8077 *
8078 * @param pVCpu The cross context virtual CPU structure.
8079 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8080 * out-of-sync. Make sure to update the required fields
8081 * before using them.
8082 */
8083DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8084{
8085 NOREF(pMixedCtx);
8086 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8087 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8088 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8089}
8090
8091
8092/**
8093 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8094 *
8095 * @param pVCpu The cross context virtual CPU structure.
8096 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8097 * out-of-sync. Make sure to update the required fields
8098 * before using them.
8099 * @param cbInstr The value of RIP that is to be pushed on the guest
8100 * stack.
8101 */
8102DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8103{
8104 NOREF(pMixedCtx);
8105 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8106 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8107 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8108}
8109
8110
8111/**
8112 * Injects a general-protection (\#GP) fault into the VM.
8113 *
8114 * @returns Strict VBox status code (i.e. informational status codes too).
8115 * @param pVCpu The cross context virtual CPU structure.
8116 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8117 * out-of-sync. Make sure to update the required fields
8118 * before using them.
8119 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8120 * mode, i.e. in real-mode it's not valid).
8121 * @param u32ErrorCode The error code associated with the \#GP.
8122 * @param fStepping Whether we're running in
8123 * hmR0VmxRunGuestCodeStep() and should return
8124 * VINF_EM_DBG_STEPPED if the event is injected
8125 * directly (register modified by us, not by
8126 * hardware on VM-entry).
8127 * @param puIntrState Pointer to the current guest interruptibility-state.
8128 * This interruptibility-state will be updated if
8129 * necessary. This cannot not be NULL.
8130 */
8131DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8132 bool fStepping, uint32_t *puIntrState)
8133{
8134 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8135 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8136 if (fErrorCodeValid)
8137 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8138 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8139 fStepping, puIntrState);
8140}
8141
8142
8143#if 0 /* unused */
8144/**
8145 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8146 * VM.
8147 *
8148 * @param pVCpu The cross context virtual CPU structure.
8149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8150 * out-of-sync. Make sure to update the required fields
8151 * before using them.
8152 * @param u32ErrorCode The error code associated with the \#GP.
8153 */
8154DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8155{
8156 NOREF(pMixedCtx);
8157 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8158 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8159 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8160 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8161}
8162#endif /* unused */
8163
8164
8165/**
8166 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8167 *
8168 * @param pVCpu The cross context virtual CPU structure.
8169 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8170 * out-of-sync. Make sure to update the required fields
8171 * before using them.
8172 * @param uVector The software interrupt vector number.
8173 * @param cbInstr The value of RIP that is to be pushed on the guest
8174 * stack.
8175 */
8176DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8177{
8178 NOREF(pMixedCtx);
8179 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8180 if ( uVector == X86_XCPT_BP
8181 || uVector == X86_XCPT_OF)
8182 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8183 else
8184 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8185 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8186}
8187
8188
8189/**
8190 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8191 * stack.
8192 *
8193 * @returns Strict VBox status code (i.e. informational status codes too).
8194 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8195 * @param pVM The cross context VM structure.
8196 * @param pMixedCtx Pointer to the guest-CPU context.
8197 * @param uValue The value to push to the guest stack.
8198 */
8199DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8200{
8201 /*
8202 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8203 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8204 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8205 */
8206 if (pMixedCtx->sp == 1)
8207 return VINF_EM_RESET;
8208 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8209 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8210 AssertRC(rc);
8211 return rc;
8212}
8213
8214
8215/**
8216 * Injects an event into the guest upon VM-entry by updating the relevant fields
8217 * in the VM-entry area in the VMCS.
8218 *
8219 * @returns Strict VBox status code (i.e. informational status codes too).
8220 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8221 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8222 *
8223 * @param pVCpu The cross context virtual CPU structure.
8224 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8225 * be out-of-sync. Make sure to update the required
8226 * fields before using them.
8227 * @param u64IntInfo The VM-entry interruption-information field.
8228 * @param cbInstr The VM-entry instruction length in bytes (for
8229 * software interrupts, exceptions and privileged
8230 * software exceptions).
8231 * @param u32ErrCode The VM-entry exception error code.
8232 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8233 * @param puIntrState Pointer to the current guest interruptibility-state.
8234 * This interruptibility-state will be updated if
8235 * necessary. This cannot not be NULL.
8236 * @param fStepping Whether we're running in
8237 * hmR0VmxRunGuestCodeStep() and should return
8238 * VINF_EM_DBG_STEPPED if the event is injected
8239 * directly (register modified by us, not by
8240 * hardware on VM-entry).
8241 *
8242 * @remarks Requires CR0!
8243 */
8244static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8245 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8246 uint32_t *puIntrState)
8247{
8248 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8249 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8250 Assert(puIntrState);
8251 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8252
8253 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8254 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8255
8256#ifdef VBOX_STRICT
8257 /*
8258 * Validate the error-code-valid bit for hardware exceptions.
8259 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8260 */
8261 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8262 && !CPUMIsGuestInRealModeEx(pMixedCtx))
8263 {
8264 switch (uVector)
8265 {
8266 case X86_XCPT_PF:
8267 case X86_XCPT_DF:
8268 case X86_XCPT_TS:
8269 case X86_XCPT_NP:
8270 case X86_XCPT_SS:
8271 case X86_XCPT_GP:
8272 case X86_XCPT_AC:
8273 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8274 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8275 RT_FALL_THRU();
8276 default:
8277 break;
8278 }
8279 }
8280#endif
8281
8282 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8283 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8284 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8285
8286 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8287
8288 /* We require CR0 to check if the guest is in real-mode. */
8289 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8290 AssertRCReturn(rc, rc);
8291
8292 /*
8293 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8294 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8295 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8296 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8297 */
8298 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8299 {
8300 PVM pVM = pVCpu->CTX_SUFF(pVM);
8301 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8302 {
8303 Assert(PDMVmmDevHeapIsEnabled(pVM));
8304 Assert(pVM->hm.s.vmx.pRealModeTSS);
8305
8306 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8307 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8308 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8309 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8310 AssertRCReturn(rc, rc);
8311 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8312
8313 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8314 size_t const cbIdtEntry = sizeof(X86IDTR16);
8315 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8316 {
8317 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8318 if (uVector == X86_XCPT_DF)
8319 return VINF_EM_RESET;
8320
8321 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8322 if (uVector == X86_XCPT_GP)
8323 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8324
8325 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8326 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8327 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8328 fStepping, puIntrState);
8329 }
8330
8331 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8332 uint16_t uGuestIp = pMixedCtx->ip;
8333 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8334 {
8335 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8336 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8337 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8338 }
8339 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8340 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8341
8342 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8343 X86IDTR16 IdtEntry;
8344 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8345 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8346 AssertRCReturn(rc, rc);
8347
8348 /* Construct the stack frame for the interrupt/exception handler. */
8349 VBOXSTRICTRC rcStrict;
8350 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8351 if (rcStrict == VINF_SUCCESS)
8352 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8353 if (rcStrict == VINF_SUCCESS)
8354 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8355
8356 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8357 if (rcStrict == VINF_SUCCESS)
8358 {
8359 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8360 pMixedCtx->rip = IdtEntry.offSel;
8361 pMixedCtx->cs.Sel = IdtEntry.uSel;
8362 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8363 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8364 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8365 && uVector == X86_XCPT_PF)
8366 pMixedCtx->cr2 = GCPtrFaultAddress;
8367
8368 /* If any other guest-state bits are changed here, make sure to update
8369 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8370 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8371 | HM_CHANGED_GUEST_RIP
8372 | HM_CHANGED_GUEST_RFLAGS
8373 | HM_CHANGED_GUEST_RSP);
8374
8375 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8376 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8377 {
8378 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8379 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8380 Log4(("Clearing inhibition due to STI.\n"));
8381 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8382 }
8383 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8384 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8385
8386 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8387 it, if we are returning to ring-3 before executing guest code. */
8388 pVCpu->hm.s.Event.fPending = false;
8389
8390 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8391 if (fStepping)
8392 rcStrict = VINF_EM_DBG_STEPPED;
8393 }
8394 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8395 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8396 return rcStrict;
8397 }
8398
8399 /*
8400 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8401 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8402 */
8403 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8404 }
8405
8406 /* Validate. */
8407 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8408 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8409 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8410
8411 /* Inject. */
8412 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8413 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8414 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8415 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8416
8417 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8418 && uVector == X86_XCPT_PF)
8419 pMixedCtx->cr2 = GCPtrFaultAddress;
8420
8421 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8422 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8423
8424 AssertRCReturn(rc, rc);
8425 return VINF_SUCCESS;
8426}
8427
8428
8429/**
8430 * Clears the interrupt-window exiting control in the VMCS and if necessary
8431 * clears the current event in the VMCS as well.
8432 *
8433 * @returns VBox status code.
8434 * @param pVCpu The cross context virtual CPU structure.
8435 *
8436 * @remarks Use this function only to clear events that have not yet been
8437 * delivered to the guest but are injected in the VMCS!
8438 * @remarks No-long-jump zone!!!
8439 */
8440static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8441{
8442 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8443
8444 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8445 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8446
8447 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8448 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8449}
8450
8451
8452/**
8453 * Enters the VT-x session.
8454 *
8455 * @returns VBox status code.
8456 * @param pVM The cross context VM structure.
8457 * @param pVCpu The cross context virtual CPU structure.
8458 * @param pCpu Pointer to the CPU info struct.
8459 */
8460VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8461{
8462 AssertPtr(pVM);
8463 AssertPtr(pVCpu);
8464 Assert(pVM->hm.s.vmx.fSupported);
8465 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8466 NOREF(pCpu); NOREF(pVM);
8467
8468 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8469 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8470
8471#ifdef VBOX_STRICT
8472 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8473 RTCCUINTREG uHostCR4 = ASMGetCR4();
8474 if (!(uHostCR4 & X86_CR4_VMXE))
8475 {
8476 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8477 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8478 }
8479#endif
8480
8481 /*
8482 * Load the VCPU's VMCS as the current (and active) one.
8483 */
8484 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8485 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8486 if (RT_FAILURE(rc))
8487 return rc;
8488
8489 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8490 pVCpu->hm.s.fLeaveDone = false;
8491 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8492
8493 return VINF_SUCCESS;
8494}
8495
8496
8497/**
8498 * The thread-context callback (only on platforms which support it).
8499 *
8500 * @param enmEvent The thread-context event.
8501 * @param pVCpu The cross context virtual CPU structure.
8502 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8503 * @thread EMT(pVCpu)
8504 */
8505VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8506{
8507 NOREF(fGlobalInit);
8508
8509 switch (enmEvent)
8510 {
8511 case RTTHREADCTXEVENT_OUT:
8512 {
8513 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8514 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8515 VMCPU_ASSERT_EMT(pVCpu);
8516
8517 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8518
8519 /* No longjmps (logger flushes, locks) in this fragile context. */
8520 VMMRZCallRing3Disable(pVCpu);
8521 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8522
8523 /*
8524 * Restore host-state (FPU, debug etc.)
8525 */
8526 if (!pVCpu->hm.s.fLeaveDone)
8527 {
8528 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8529 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8530 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8531 pVCpu->hm.s.fLeaveDone = true;
8532 }
8533
8534 /* Leave HM context, takes care of local init (term). */
8535 int rc = HMR0LeaveCpu(pVCpu);
8536 AssertRC(rc); NOREF(rc);
8537
8538 /* Restore longjmp state. */
8539 VMMRZCallRing3Enable(pVCpu);
8540 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8541 break;
8542 }
8543
8544 case RTTHREADCTXEVENT_IN:
8545 {
8546 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8547 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8548 VMCPU_ASSERT_EMT(pVCpu);
8549
8550 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8551 VMMRZCallRing3Disable(pVCpu);
8552 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8553
8554 /* Initialize the bare minimum state required for HM. This takes care of
8555 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8556 int rc = HMR0EnterCpu(pVCpu);
8557 AssertRC(rc);
8558 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8559
8560 /* Load the active VMCS as the current one. */
8561 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8562 {
8563 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8564 AssertRC(rc); NOREF(rc);
8565 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8566 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8567 }
8568 pVCpu->hm.s.fLeaveDone = false;
8569
8570 /* Restore longjmp state. */
8571 VMMRZCallRing3Enable(pVCpu);
8572 break;
8573 }
8574
8575 default:
8576 break;
8577 }
8578}
8579
8580
8581/**
8582 * Saves the host state in the VMCS host-state.
8583 * Sets up the VM-exit MSR-load area.
8584 *
8585 * The CPU state will be loaded from these fields on every successful VM-exit.
8586 *
8587 * @returns VBox status code.
8588 * @param pVM The cross context VM structure.
8589 * @param pVCpu The cross context virtual CPU structure.
8590 *
8591 * @remarks No-long-jump zone!!!
8592 */
8593static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8594{
8595 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8596
8597 int rc = VINF_SUCCESS;
8598 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8599 {
8600 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8601 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8602
8603 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8604 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8605
8606 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8607 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8608
8609 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8610 }
8611 return rc;
8612}
8613
8614
8615/**
8616 * Saves the host state in the VMCS host-state.
8617 *
8618 * @returns VBox status code.
8619 * @param pVM The cross context VM structure.
8620 * @param pVCpu The cross context virtual CPU structure.
8621 *
8622 * @remarks No-long-jump zone!!!
8623 */
8624VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8625{
8626 AssertPtr(pVM);
8627 AssertPtr(pVCpu);
8628
8629 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8630
8631 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8632 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8633 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8634 return hmR0VmxSaveHostState(pVM, pVCpu);
8635}
8636
8637
8638/**
8639 * Loads the guest state into the VMCS guest-state area.
8640 *
8641 * The will typically be done before VM-entry when the guest-CPU state and the
8642 * VMCS state may potentially be out of sync.
8643 *
8644 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8645 * VM-entry controls.
8646 * Sets up the appropriate VMX non-root function to execute guest code based on
8647 * the guest CPU mode.
8648 *
8649 * @returns VBox strict status code.
8650 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8651 * without unrestricted guest access and the VMMDev is not presently
8652 * mapped (e.g. EFI32).
8653 *
8654 * @param pVM The cross context VM structure.
8655 * @param pVCpu The cross context virtual CPU structure.
8656 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8657 * out-of-sync. Make sure to update the required fields
8658 * before using them.
8659 *
8660 * @remarks No-long-jump zone!!!
8661 */
8662static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8663{
8664 AssertPtr(pVM);
8665 AssertPtr(pVCpu);
8666 AssertPtr(pMixedCtx);
8667 HMVMX_ASSERT_PREEMPT_SAFE();
8668
8669 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8670
8671 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8672
8673 /* Determine real-on-v86 mode. */
8674 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8675 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8676 && CPUMIsGuestInRealModeEx(pMixedCtx))
8677 {
8678 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8679 }
8680
8681 /*
8682 * Load the guest-state into the VMCS.
8683 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8684 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8685 */
8686 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8687 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8688
8689 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8690 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8691 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8692
8693 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8694 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8695 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8696
8697 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8698 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8699
8700 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8701 if (rcStrict == VINF_SUCCESS)
8702 { /* likely */ }
8703 else
8704 {
8705 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8706 return rcStrict;
8707 }
8708
8709 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8710 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8711 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8712
8713 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8714 determine we don't have to swap EFER after all. */
8715 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8716 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8717
8718 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8719 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8720
8721 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8722 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8723
8724 /*
8725 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8726 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8727 */
8728 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8729 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8730
8731 /* Clear any unused and reserved bits. */
8732 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
8733 | HM_CHANGED_GUEST_HWVIRT);
8734
8735 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8736 return rc;
8737}
8738
8739
8740/**
8741 * Loads the state shared between the host and guest into the VMCS.
8742 *
8743 * @param pVM The cross context VM structure.
8744 * @param pVCpu The cross context virtual CPU structure.
8745 * @param pCtx Pointer to the guest-CPU context.
8746 *
8747 * @remarks No-long-jump zone!!!
8748 */
8749static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8750{
8751 NOREF(pVM);
8752
8753 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8754 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8755
8756 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8757 {
8758 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8759 AssertRC(rc);
8760 }
8761
8762 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8763 {
8764 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8765 AssertRC(rc);
8766
8767 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8768 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8769 {
8770 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8771 AssertRC(rc);
8772 }
8773 }
8774
8775 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
8776 {
8777 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8778 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
8779 }
8780
8781 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8782 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
8783 {
8784 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8785 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8786 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8787 AssertRC(rc);
8788 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
8789 }
8790
8791 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8792 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8793}
8794
8795
8796/**
8797 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8798 *
8799 * @returns Strict VBox status code (i.e. informational status codes too).
8800 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8801 * without unrestricted guest access and the VMMDev is not presently
8802 * mapped (e.g. EFI32).
8803 *
8804 * @param pVM The cross context VM structure.
8805 * @param pVCpu The cross context virtual CPU structure.
8806 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8807 * out-of-sync. Make sure to update the required fields
8808 * before using them.
8809 *
8810 * @remarks No-long-jump zone!!!
8811 */
8812static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8813{
8814 HMVMX_ASSERT_PREEMPT_SAFE();
8815 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8816 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8817
8818 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8819#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8820 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8821#endif
8822
8823 /*
8824 * RIP is what changes the most often and hence if it's the only bit needing to be
8825 * updated, we shall handle it early for performance reasons.
8826 */
8827 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8828 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8829 {
8830 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8831 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8832 { /* likely */}
8833 else
8834 {
8835 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8836 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8837 }
8838 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8839 }
8840 else if (HMCPU_CF_VALUE(pVCpu))
8841 {
8842 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8843 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8844 { /* likely */}
8845 else
8846 {
8847 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8848 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8849 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8850 return rcStrict;
8851 }
8852 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8853 }
8854
8855 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8856 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8857 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8858 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8859 return rcStrict;
8860}
8861
8862
8863/**
8864 * Does the preparations before executing guest code in VT-x.
8865 *
8866 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8867 * recompiler/IEM. We must be cautious what we do here regarding committing
8868 * guest-state information into the VMCS assuming we assuredly execute the
8869 * guest in VT-x mode.
8870 *
8871 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8872 * the common-state (TRPM/forceflags), we must undo those changes so that the
8873 * recompiler/IEM can (and should) use them when it resumes guest execution.
8874 * Otherwise such operations must be done when we can no longer exit to ring-3.
8875 *
8876 * @returns Strict VBox status code (i.e. informational status codes too).
8877 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8878 * have been disabled.
8879 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8880 * double-fault into the guest.
8881 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8882 * dispatched directly.
8883 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8884 *
8885 * @param pVM The cross context VM structure.
8886 * @param pVCpu The cross context virtual CPU structure.
8887 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8888 * out-of-sync. Make sure to update the required fields
8889 * before using them.
8890 * @param pVmxTransient Pointer to the VMX transient structure.
8891 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8892 * us ignore some of the reasons for returning to
8893 * ring-3, and return VINF_EM_DBG_STEPPED if event
8894 * dispatching took place.
8895 */
8896static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8897{
8898 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8899
8900#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8901 PGMRZDynMapFlushAutoSet(pVCpu);
8902#endif
8903
8904 /* Check force flag actions that might require us to go back to ring-3. */
8905 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8906 if (rcStrict == VINF_SUCCESS)
8907 { /* FFs doesn't get set all the time. */ }
8908 else
8909 return rcStrict;
8910
8911 /*
8912 * Setup the virtualized-APIC accesses.
8913 *
8914 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8915 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8916 *
8917 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8918 */
8919 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8920 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8921 && PDMHasApic(pVM))
8922 {
8923 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8924 Assert(u64MsrApicBase);
8925 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8926
8927 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8928
8929 /* Unalias any existing mapping. */
8930 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8931 AssertRCReturn(rc, rc);
8932
8933 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8934 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8935 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8936 AssertRCReturn(rc, rc);
8937
8938 /* Update the per-VCPU cache of the APIC base MSR. */
8939 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8940 }
8941
8942 if (TRPMHasTrap(pVCpu))
8943 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8944 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8945
8946 /*
8947 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8948 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8949 */
8950 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8951 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8952 { /* likely */ }
8953 else
8954 {
8955 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8956 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8957 return rcStrict;
8958 }
8959
8960 /*
8961 * No longjmps to ring-3 from this point on!!!
8962 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8963 * This also disables flushing of the R0-logger instance (if any).
8964 */
8965 VMMRZCallRing3Disable(pVCpu);
8966
8967 /*
8968 * Load the guest state bits.
8969 *
8970 * We cannot perform longjmps while loading the guest state because we do not preserve the
8971 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8972 * CPU migration.
8973 *
8974 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8975 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8976 * Hence, loading of the guest state needs to be done -after- injection of events.
8977 */
8978 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8979 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8980 { /* likely */ }
8981 else
8982 {
8983 VMMRZCallRing3Enable(pVCpu);
8984 return rcStrict;
8985 }
8986
8987 /*
8988 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8989 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8990 *
8991 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8992 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8993 *
8994 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8995 * executing guest code.
8996 */
8997 pVmxTransient->fEFlags = ASMIntDisableFlags();
8998
8999 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
9000 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
9001 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
9002 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
9003 {
9004 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
9005 {
9006 pVCpu->hm.s.Event.fPending = false;
9007
9008 /*
9009 * We've injected any pending events. This is really the point of no return (to ring-3).
9010 *
9011 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
9012 * returns from this function, so don't enable them here.
9013 */
9014 return VINF_SUCCESS;
9015 }
9016
9017 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9018 rcStrict = VINF_EM_RAW_INTERRUPT;
9019 }
9020 else
9021 {
9022 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9023 rcStrict = VINF_EM_RAW_TO_R3;
9024 }
9025
9026 ASMSetFlags(pVmxTransient->fEFlags);
9027 VMMRZCallRing3Enable(pVCpu);
9028
9029 return rcStrict;
9030}
9031
9032
9033/**
9034 * Prepares to run guest code in VT-x and we've committed to doing so. This
9035 * means there is no backing out to ring-3 or anywhere else at this
9036 * point.
9037 *
9038 * @param pVM The cross context VM structure.
9039 * @param pVCpu The cross context virtual CPU structure.
9040 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9041 * out-of-sync. Make sure to update the required fields
9042 * before using them.
9043 * @param pVmxTransient Pointer to the VMX transient structure.
9044 *
9045 * @remarks Called with preemption disabled.
9046 * @remarks No-long-jump zone!!!
9047 */
9048static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9049{
9050 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9051 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9052 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9053
9054 /*
9055 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9056 */
9057 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9058 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9059
9060 if (!CPUMIsGuestFPUStateActive(pVCpu))
9061 {
9062 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9063 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9064 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9065 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9066 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9067 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9068 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9069 }
9070
9071 /*
9072 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9073 */
9074 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9075 && pVCpu->hm.s.vmx.cMsrs > 0)
9076 {
9077 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9078 }
9079
9080 /*
9081 * Load the host state bits as we may've been preempted (only happens when
9082 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9083 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9084 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9085 * See @bugref{8432}.
9086 */
9087 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9088 {
9089 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9090 AssertRC(rc);
9091 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9092 }
9093 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9094
9095 /*
9096 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9097 */
9098 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9099 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9100 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9101
9102 /* Store status of the shared guest-host state at the time of VM-entry. */
9103#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9104 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9105 {
9106 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9107 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9108 }
9109 else
9110#endif
9111 {
9112 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9113 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9114 }
9115
9116 /*
9117 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9118 */
9119 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9120 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9121
9122 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9123 RTCPUID idCurrentCpu = pCpu->idCpu;
9124 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9125 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9126 {
9127 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9128 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9129 }
9130
9131 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9132 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9133 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9134 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9135
9136 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9137
9138 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9139 to start executing. */
9140
9141 /*
9142 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9143 */
9144 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9145 {
9146 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9147 {
9148 bool fMsrUpdated;
9149 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9150 AssertRC(rc2);
9151 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9152
9153 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9154 &fMsrUpdated);
9155 AssertRC(rc2);
9156 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9157
9158 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9159 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9160 }
9161 else
9162 {
9163 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9164 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9165 }
9166 }
9167
9168 if (pVM->cpum.ro.GuestFeatures.fIbrs)
9169 {
9170 bool fMsrUpdated;
9171 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9172 AssertRC(rc2);
9173 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9174
9175 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9176 &fMsrUpdated);
9177 AssertRC(rc2);
9178 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9179 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9180 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9181 }
9182
9183#ifdef VBOX_STRICT
9184 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9185 hmR0VmxCheckHostEferMsr(pVCpu);
9186 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9187#endif
9188#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9189 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9190 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9191 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9192#endif
9193}
9194
9195
9196/**
9197 * Performs some essential restoration of state after running guest code in
9198 * VT-x.
9199 *
9200 * @param pVM The cross context VM structure.
9201 * @param pVCpu The cross context virtual CPU structure.
9202 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9203 * out-of-sync. Make sure to update the required fields
9204 * before using them.
9205 * @param pVmxTransient Pointer to the VMX transient structure.
9206 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9207 *
9208 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9209 *
9210 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9211 * unconditionally when it is safe to do so.
9212 */
9213static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9214{
9215 NOREF(pVM);
9216 uint64_t uHostTsc = ASMReadTSC();
9217
9218 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9219
9220 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9221 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9222 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9223 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9224 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9225 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9226
9227 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9228 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TSCOffset);
9229
9230 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9231 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9232 Assert(!ASMIntAreEnabled());
9233 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9234
9235#if HC_ARCH_BITS == 64
9236 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9237#endif
9238#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9239 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9240 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9241 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9242#else
9243 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9244#endif
9245#ifdef VBOX_STRICT
9246 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9247#endif
9248 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9249
9250 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9251 uint32_t uExitReason;
9252 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9253 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9254 AssertRC(rc);
9255 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9256 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9257
9258 if (rcVMRun == VINF_SUCCESS)
9259 {
9260 /*
9261 * Update the VM-exit history array here even if the VM-entry failed due to:
9262 * - Invalid guest state.
9263 * - MSR loading.
9264 * - Machine-check event.
9265 *
9266 * In any of the above cases we will still have a "valid" VM-exit reason
9267 * despite @a fVMEntryFailed being false.
9268 *
9269 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9270 *
9271 * Note! We don't have CS or RIP at this point. Will probably address that later
9272 * by amending the history entry added here.
9273 */
9274 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FLAGS_AND_TYPE(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
9275 UINT64_MAX, uHostTsc);
9276
9277 if (!pVmxTransient->fVMEntryFailed)
9278 {
9279 /** @todo We can optimize this by only syncing with our force-flags when
9280 * really needed and keeping the VMCS state as it is for most
9281 * VM-exits. */
9282 /* Update the guest interruptibility-state from the VMCS. */
9283 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9284
9285 /*
9286 * Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9287 * as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9288 * on the longjmp path to ring-3 while saving the (rest of) the guest state,
9289 * see @bugref{6208#c63}.
9290 */
9291 VMMRZCallRing3Enable(pVCpu);
9292
9293#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9294 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9295 AssertRC(rc);
9296#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9297 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9298 AssertRC(rc);
9299#endif
9300
9301 /*
9302 * Sync the TPR shadow with our APIC state.
9303 */
9304 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9305 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9306 {
9307 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9308 AssertRC(rc);
9309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9310 }
9311
9312 return;
9313 }
9314 }
9315 else
9316 {
9317 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9318 pVmxTransient->fVMEntryFailed));
9319 }
9320
9321 VMMRZCallRing3Enable(pVCpu);
9322}
9323
9324
9325/**
9326 * Runs the guest code using VT-x the normal way.
9327 *
9328 * @returns VBox status code.
9329 * @param pVM The cross context VM structure.
9330 * @param pVCpu The cross context virtual CPU structure.
9331 * @param pCtx Pointer to the guest-CPU context.
9332 *
9333 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9334 */
9335static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9336{
9337 VMXTRANSIENT VmxTransient;
9338 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9339 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9340 uint32_t cLoops = 0;
9341
9342 for (;; cLoops++)
9343 {
9344 Assert(!HMR0SuspendPending());
9345 HMVMX_ASSERT_CPU_SAFE();
9346
9347 /* Preparatory work for running guest code, this may force us to return
9348 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9349 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9350 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9351 if (rcStrict != VINF_SUCCESS)
9352 break;
9353
9354 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9355 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9356 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9357
9358 /* Restore any residual host-state and save any bits shared between host
9359 and guest into the guest-CPU state. Re-enables interrupts! */
9360 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9361
9362 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9363 if (RT_SUCCESS(rcRun))
9364 { /* very likely */ }
9365 else
9366 {
9367 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9368 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9369 return rcRun;
9370 }
9371
9372 /* Profile the VM-exit. */
9373 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9375 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9376 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9377 HMVMX_START_EXIT_DISPATCH_PROF();
9378
9379 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9380
9381 /* Handle the VM-exit. */
9382#ifdef HMVMX_USE_FUNCTION_TABLE
9383 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9384#else
9385 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9386#endif
9387 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9388 if (rcStrict == VINF_SUCCESS)
9389 {
9390 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9391 continue; /* likely */
9392 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9393 rcStrict = VINF_EM_RAW_INTERRUPT;
9394 }
9395 break;
9396 }
9397
9398 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9399 return rcStrict;
9400}
9401
9402
9403
9404/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9405 * probes.
9406 *
9407 * The following few functions and associated structure contains the bloat
9408 * necessary for providing detailed debug events and dtrace probes as well as
9409 * reliable host side single stepping. This works on the principle of
9410 * "subclassing" the normal execution loop and workers. We replace the loop
9411 * method completely and override selected helpers to add necessary adjustments
9412 * to their core operation.
9413 *
9414 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9415 * any performance for debug and analysis features.
9416 *
9417 * @{
9418 */
9419
9420/**
9421 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9422 * the debug run loop.
9423 */
9424typedef struct VMXRUNDBGSTATE
9425{
9426 /** The RIP we started executing at. This is for detecting that we stepped. */
9427 uint64_t uRipStart;
9428 /** The CS we started executing with. */
9429 uint16_t uCsStart;
9430
9431 /** Whether we've actually modified the 1st execution control field. */
9432 bool fModifiedProcCtls : 1;
9433 /** Whether we've actually modified the 2nd execution control field. */
9434 bool fModifiedProcCtls2 : 1;
9435 /** Whether we've actually modified the exception bitmap. */
9436 bool fModifiedXcptBitmap : 1;
9437
9438 /** We desire the modified the CR0 mask to be cleared. */
9439 bool fClearCr0Mask : 1;
9440 /** We desire the modified the CR4 mask to be cleared. */
9441 bool fClearCr4Mask : 1;
9442 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9443 uint32_t fCpe1Extra;
9444 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9445 uint32_t fCpe1Unwanted;
9446 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9447 uint32_t fCpe2Extra;
9448 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9449 uint32_t bmXcptExtra;
9450 /** The sequence number of the Dtrace provider settings the state was
9451 * configured against. */
9452 uint32_t uDtraceSettingsSeqNo;
9453 /** VM-exits to check (one bit per VM-exit). */
9454 uint32_t bmExitsToCheck[3];
9455
9456 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9457 uint32_t fProcCtlsInitial;
9458 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9459 uint32_t fProcCtls2Initial;
9460 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9461 uint32_t bmXcptInitial;
9462} VMXRUNDBGSTATE;
9463AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9464typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9465
9466
9467/**
9468 * Initializes the VMXRUNDBGSTATE structure.
9469 *
9470 * @param pVCpu The cross context virtual CPU structure of the
9471 * calling EMT.
9472 * @param pCtx The CPU register context to go with @a pVCpu.
9473 * @param pDbgState The structure to initialize.
9474 */
9475DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9476{
9477 pDbgState->uRipStart = pCtx->rip;
9478 pDbgState->uCsStart = pCtx->cs.Sel;
9479
9480 pDbgState->fModifiedProcCtls = false;
9481 pDbgState->fModifiedProcCtls2 = false;
9482 pDbgState->fModifiedXcptBitmap = false;
9483 pDbgState->fClearCr0Mask = false;
9484 pDbgState->fClearCr4Mask = false;
9485 pDbgState->fCpe1Extra = 0;
9486 pDbgState->fCpe1Unwanted = 0;
9487 pDbgState->fCpe2Extra = 0;
9488 pDbgState->bmXcptExtra = 0;
9489 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9490 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9491 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9492}
9493
9494
9495/**
9496 * Updates the VMSC fields with changes requested by @a pDbgState.
9497 *
9498 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9499 * immediately before executing guest code, i.e. when interrupts are disabled.
9500 * We don't check status codes here as we cannot easily assert or return in the
9501 * latter case.
9502 *
9503 * @param pVCpu The cross context virtual CPU structure.
9504 * @param pDbgState The debug state.
9505 */
9506DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9507{
9508 /*
9509 * Ensure desired flags in VMCS control fields are set.
9510 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9511 *
9512 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9513 * there should be no stale data in pCtx at this point.
9514 */
9515 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9516 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9517 {
9518 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9519 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9520 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9521 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9522 pDbgState->fModifiedProcCtls = true;
9523 }
9524
9525 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9526 {
9527 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9528 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9529 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9530 pDbgState->fModifiedProcCtls2 = true;
9531 }
9532
9533 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9534 {
9535 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9536 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9537 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9538 pDbgState->fModifiedXcptBitmap = true;
9539 }
9540
9541 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9542 {
9543 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9544 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9545 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9546 }
9547
9548 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9549 {
9550 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9551 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9552 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9553 }
9554}
9555
9556
9557DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9558{
9559 /*
9560 * Restore VM-exit control settings as we may not reenter this function the
9561 * next time around.
9562 */
9563 /* We reload the initial value, trigger what we can of recalculations the
9564 next time around. From the looks of things, that's all that's required atm. */
9565 if (pDbgState->fModifiedProcCtls)
9566 {
9567 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9568 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9569 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9570 AssertRCReturn(rc2, rc2);
9571 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9572 }
9573
9574 /* We're currently the only ones messing with this one, so just restore the
9575 cached value and reload the field. */
9576 if ( pDbgState->fModifiedProcCtls2
9577 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9578 {
9579 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9580 AssertRCReturn(rc2, rc2);
9581 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9582 }
9583
9584 /* If we've modified the exception bitmap, we restore it and trigger
9585 reloading and partial recalculation the next time around. */
9586 if (pDbgState->fModifiedXcptBitmap)
9587 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9588
9589 return rcStrict;
9590}
9591
9592
9593/**
9594 * Configures VM-exit controls for current DBGF and DTrace settings.
9595 *
9596 * This updates @a pDbgState and the VMCS execution control fields to reflect
9597 * the necessary VM-exits demanded by DBGF and DTrace.
9598 *
9599 * @param pVM The cross context VM structure.
9600 * @param pVCpu The cross context virtual CPU structure.
9601 * @param pCtx Pointer to the guest-CPU context.
9602 * @param pDbgState The debug state.
9603 * @param pVmxTransient Pointer to the VMX transient structure. May update
9604 * fUpdateTscOffsettingAndPreemptTimer.
9605 */
9606static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9607 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9608{
9609 /*
9610 * Take down the dtrace serial number so we can spot changes.
9611 */
9612 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9613 ASMCompilerBarrier();
9614
9615 /*
9616 * We'll rebuild most of the middle block of data members (holding the
9617 * current settings) as we go along here, so start by clearing it all.
9618 */
9619 pDbgState->bmXcptExtra = 0;
9620 pDbgState->fCpe1Extra = 0;
9621 pDbgState->fCpe1Unwanted = 0;
9622 pDbgState->fCpe2Extra = 0;
9623 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9624 pDbgState->bmExitsToCheck[i] = 0;
9625
9626 /*
9627 * Software interrupts (INT XXh) - no idea how to trigger these...
9628 */
9629 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9630 || VBOXVMM_INT_SOFTWARE_ENABLED())
9631 {
9632 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9633 }
9634
9635 /*
9636 * INT3 breakpoints - triggered by #BP exceptions.
9637 */
9638 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9639 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9640
9641 /*
9642 * Exception bitmap and XCPT events+probes.
9643 */
9644 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9645 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9646 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9647
9648 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9649 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9650 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9651 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9652 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9653 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9654 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9655 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9656 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9657 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9658 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9659 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9660 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9661 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9662 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9663 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9664 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9665 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9666
9667 if (pDbgState->bmXcptExtra)
9668 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9669
9670 /*
9671 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9672 *
9673 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9674 * So, when adding/changing/removing please don't forget to update it.
9675 *
9676 * Some of the macros are picking up local variables to save horizontal space,
9677 * (being able to see it in a table is the lesser evil here).
9678 */
9679#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9680 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9681 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9682#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9683 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9684 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9685 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9686 } else do { } while (0)
9687#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9688 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9689 { \
9690 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9691 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9692 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9693 } else do { } while (0)
9694#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9695 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9696 { \
9697 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9698 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9699 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9700 } else do { } while (0)
9701#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9702 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9703 { \
9704 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9705 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9706 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9707 } else do { } while (0)
9708
9709 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9710 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9711 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9712 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9713 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9714
9715 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9716 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9717 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9718 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9719 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9720 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9721 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9722 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9723 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9724 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9725 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9726 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9727 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9728 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9729 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9730 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9731 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9732 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9733 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9734 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9735 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9736 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9737 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9738 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9739 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9740 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9741 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9742 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9743 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9744 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9745 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9746 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9747 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9748 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9749 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9750 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9751
9752 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9753 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9754 {
9755 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9756 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9757 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9758 AssertRC(rc2);
9759
9760#if 0 /** @todo fix me */
9761 pDbgState->fClearCr0Mask = true;
9762 pDbgState->fClearCr4Mask = true;
9763#endif
9764 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9765 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9766 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9767 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9768 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9769 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9770 require clearing here and in the loop if we start using it. */
9771 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9772 }
9773 else
9774 {
9775 if (pDbgState->fClearCr0Mask)
9776 {
9777 pDbgState->fClearCr0Mask = false;
9778 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9779 }
9780 if (pDbgState->fClearCr4Mask)
9781 {
9782 pDbgState->fClearCr4Mask = false;
9783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9784 }
9785 }
9786 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9787 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9788
9789 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9790 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9791 {
9792 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9793 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9794 }
9795 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9796 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9797
9798 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9799 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9800 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9801 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9802 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9803 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9804 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9805 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9806#if 0 /** @todo too slow, fix handler. */
9807 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9808#endif
9809 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9810
9811 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9812 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9813 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9814 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9815 {
9816 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9817 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9818 }
9819 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9820 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9821 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9822 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9823
9824 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9825 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9826 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9827 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9828 {
9829 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9830 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9831 }
9832 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9833 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9834 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9835 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9836
9837 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9838 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9839 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9840 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9841 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9842 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9843 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9844 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9845 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9846 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9847 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9848 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9849 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9850 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9851 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9852 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9853 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9854 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9855 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9856 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9857 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9858 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9859
9860#undef IS_EITHER_ENABLED
9861#undef SET_ONLY_XBM_IF_EITHER_EN
9862#undef SET_CPE1_XBM_IF_EITHER_EN
9863#undef SET_CPEU_XBM_IF_EITHER_EN
9864#undef SET_CPE2_XBM_IF_EITHER_EN
9865
9866 /*
9867 * Sanitize the control stuff.
9868 */
9869 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9870 if (pDbgState->fCpe2Extra)
9871 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9872 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9873 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9874 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9875 {
9876 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9877 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9878 }
9879
9880 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9881 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9882 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9883 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9884}
9885
9886
9887/**
9888 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9889 * appropriate.
9890 *
9891 * The caller has checked the VM-exit against the
9892 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9893 * already, so we don't have to do that either.
9894 *
9895 * @returns Strict VBox status code (i.e. informational status codes too).
9896 * @param pVM The cross context VM structure.
9897 * @param pVCpu The cross context virtual CPU structure.
9898 * @param pMixedCtx Pointer to the guest-CPU context.
9899 * @param pVmxTransient Pointer to the VMX-transient structure.
9900 * @param uExitReason The VM-exit reason.
9901 *
9902 * @remarks The name of this function is displayed by dtrace, so keep it short
9903 * and to the point. No longer than 33 chars long, please.
9904 */
9905static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9906 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9907{
9908 /*
9909 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9910 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9911 *
9912 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9913 * does. Must add/change/remove both places. Same ordering, please.
9914 *
9915 * Added/removed events must also be reflected in the next section
9916 * where we dispatch dtrace events.
9917 */
9918 bool fDtrace1 = false;
9919 bool fDtrace2 = false;
9920 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9921 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9922 uint32_t uEventArg = 0;
9923#define SET_EXIT(a_EventSubName) \
9924 do { \
9925 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9926 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9927 } while (0)
9928#define SET_BOTH(a_EventSubName) \
9929 do { \
9930 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9931 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9932 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9933 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9934 } while (0)
9935 switch (uExitReason)
9936 {
9937 case VMX_EXIT_MTF:
9938 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9939
9940 case VMX_EXIT_XCPT_OR_NMI:
9941 {
9942 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9943 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9944 {
9945 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9946 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9947 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9948 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9949 {
9950 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9951 {
9952 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9953 uEventArg = pVmxTransient->uExitIntErrorCode;
9954 }
9955 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9956 switch (enmEvent1)
9957 {
9958 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9959 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9960 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9961 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9962 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9963 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9964 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9965 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9966 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9967 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9968 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9969 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9970 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9971 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9972 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9973 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9974 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9975 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9976 default: break;
9977 }
9978 }
9979 else
9980 AssertFailed();
9981 break;
9982
9983 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9984 uEventArg = idxVector;
9985 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9986 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9987 break;
9988 }
9989 break;
9990 }
9991
9992 case VMX_EXIT_TRIPLE_FAULT:
9993 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9994 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9995 break;
9996 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9997 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9998 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9999 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
10000 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
10001
10002 /* Instruction specific VM-exits: */
10003 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10004 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10005 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10006 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10007 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10008 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10009 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10010 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10011 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10012 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10013 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10014 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10015 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10016 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10017 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10018 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10019 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10020 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10021 case VMX_EXIT_MOV_CRX:
10022 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10023/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10024* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10025 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10026 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10027 SET_BOTH(CRX_READ);
10028 else
10029 SET_BOTH(CRX_WRITE);
10030 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10031 break;
10032 case VMX_EXIT_MOV_DRX:
10033 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10034 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10035 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10036 SET_BOTH(DRX_READ);
10037 else
10038 SET_BOTH(DRX_WRITE);
10039 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10040 break;
10041 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10042 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10043 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10044 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10045 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10046 case VMX_EXIT_XDTR_ACCESS:
10047 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10048 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10049 {
10050 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10051 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10052 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10053 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10054 }
10055 break;
10056
10057 case VMX_EXIT_TR_ACCESS:
10058 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10059 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10060 {
10061 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10062 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10063 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10064 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10065 }
10066 break;
10067
10068 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10069 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10070 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10071 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10072 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10073 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10074 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10075 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10076 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10077 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10078 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10079
10080 /* Events that aren't relevant at this point. */
10081 case VMX_EXIT_EXT_INT:
10082 case VMX_EXIT_INT_WINDOW:
10083 case VMX_EXIT_NMI_WINDOW:
10084 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10085 case VMX_EXIT_PREEMPT_TIMER:
10086 case VMX_EXIT_IO_INSTR:
10087 break;
10088
10089 /* Errors and unexpected events. */
10090 case VMX_EXIT_INIT_SIGNAL:
10091 case VMX_EXIT_SIPI:
10092 case VMX_EXIT_IO_SMI:
10093 case VMX_EXIT_SMI:
10094 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10095 case VMX_EXIT_ERR_MSR_LOAD:
10096 case VMX_EXIT_ERR_MACHINE_CHECK:
10097 break;
10098
10099 default:
10100 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10101 break;
10102 }
10103#undef SET_BOTH
10104#undef SET_EXIT
10105
10106 /*
10107 * Dtrace tracepoints go first. We do them here at once so we don't
10108 * have to copy the guest state saving and stuff a few dozen times.
10109 * Down side is that we've got to repeat the switch, though this time
10110 * we use enmEvent since the probes are a subset of what DBGF does.
10111 */
10112 if (fDtrace1 || fDtrace2)
10113 {
10114 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10115 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10116 switch (enmEvent1)
10117 {
10118 /** @todo consider which extra parameters would be helpful for each probe. */
10119 case DBGFEVENT_END: break;
10120 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10121 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10122 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10123 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10124 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10125 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10126 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10127 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10128 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10129 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10130 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10131 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10132 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10133 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10134 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10135 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10136 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10137 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10138 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10139 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10140 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10141 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10142 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10143 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10144 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10145 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10146 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10147 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10148 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10149 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10150 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10151 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10152 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10153 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10154 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10155 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10156 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10157 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10158 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10159 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10160 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10161 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10162 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10163 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10164 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10165 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10166 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10167 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10168 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10169 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10170 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10171 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10172 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10173 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10178 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10179 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10180 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10181 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10182 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10183 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10184 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10185 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10186 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10187 }
10188 switch (enmEvent2)
10189 {
10190 /** @todo consider which extra parameters would be helpful for each probe. */
10191 case DBGFEVENT_END: break;
10192 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10194 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10197 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10198 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10199 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10200 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10201 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10202 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10203 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10204 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10205 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10206 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10207 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10208 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10209 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10210 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10211 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10212 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10213 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10214 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10215 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10216 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10217 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10218 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10219 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10220 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10221 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10222 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10223 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10224 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10225 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10226 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10227 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10228 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10229 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10230 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10231 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10232 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10233 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10234 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10235 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10236 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10237 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10238 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10239 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10240 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10241 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10242 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10243 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10244 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10245 }
10246 }
10247
10248 /*
10249 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10250 * the DBGF call will do a full check).
10251 *
10252 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10253 * Note! If we have to events, we prioritize the first, i.e. the instruction
10254 * one, in order to avoid event nesting.
10255 */
10256 if ( enmEvent1 != DBGFEVENT_END
10257 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10258 {
10259 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10260 if (rcStrict != VINF_SUCCESS)
10261 return rcStrict;
10262 }
10263 else if ( enmEvent2 != DBGFEVENT_END
10264 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10265 {
10266 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10267 if (rcStrict != VINF_SUCCESS)
10268 return rcStrict;
10269 }
10270
10271 return VINF_SUCCESS;
10272}
10273
10274
10275/**
10276 * Single-stepping VM-exit filtering.
10277 *
10278 * This is preprocessing the VM-exits and deciding whether we've gotten far
10279 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10280 * handling is performed.
10281 *
10282 * @returns Strict VBox status code (i.e. informational status codes too).
10283 * @param pVM The cross context VM structure.
10284 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10285 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10286 * out-of-sync. Make sure to update the required
10287 * fields before using them.
10288 * @param pVmxTransient Pointer to the VMX-transient structure.
10289 * @param uExitReason The VM-exit reason.
10290 * @param pDbgState The debug state.
10291 */
10292DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10293 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10294{
10295 /*
10296 * Expensive (saves context) generic dtrace VM-exit probe.
10297 */
10298 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10299 { /* more likely */ }
10300 else
10301 {
10302 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10303 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10304 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10305 }
10306
10307 /*
10308 * Check for host NMI, just to get that out of the way.
10309 */
10310 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10311 { /* normally likely */ }
10312 else
10313 {
10314 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10315 AssertRCReturn(rc2, rc2);
10316 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10317 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10318 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10319 }
10320
10321 /*
10322 * Check for single stepping event if we're stepping.
10323 */
10324 if (pVCpu->hm.s.fSingleInstruction)
10325 {
10326 switch (uExitReason)
10327 {
10328 case VMX_EXIT_MTF:
10329 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10330
10331 /* Various events: */
10332 case VMX_EXIT_XCPT_OR_NMI:
10333 case VMX_EXIT_EXT_INT:
10334 case VMX_EXIT_TRIPLE_FAULT:
10335 case VMX_EXIT_INT_WINDOW:
10336 case VMX_EXIT_NMI_WINDOW:
10337 case VMX_EXIT_TASK_SWITCH:
10338 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10339 case VMX_EXIT_APIC_ACCESS:
10340 case VMX_EXIT_EPT_VIOLATION:
10341 case VMX_EXIT_EPT_MISCONFIG:
10342 case VMX_EXIT_PREEMPT_TIMER:
10343
10344 /* Instruction specific VM-exits: */
10345 case VMX_EXIT_CPUID:
10346 case VMX_EXIT_GETSEC:
10347 case VMX_EXIT_HLT:
10348 case VMX_EXIT_INVD:
10349 case VMX_EXIT_INVLPG:
10350 case VMX_EXIT_RDPMC:
10351 case VMX_EXIT_RDTSC:
10352 case VMX_EXIT_RSM:
10353 case VMX_EXIT_VMCALL:
10354 case VMX_EXIT_VMCLEAR:
10355 case VMX_EXIT_VMLAUNCH:
10356 case VMX_EXIT_VMPTRLD:
10357 case VMX_EXIT_VMPTRST:
10358 case VMX_EXIT_VMREAD:
10359 case VMX_EXIT_VMRESUME:
10360 case VMX_EXIT_VMWRITE:
10361 case VMX_EXIT_VMXOFF:
10362 case VMX_EXIT_VMXON:
10363 case VMX_EXIT_MOV_CRX:
10364 case VMX_EXIT_MOV_DRX:
10365 case VMX_EXIT_IO_INSTR:
10366 case VMX_EXIT_RDMSR:
10367 case VMX_EXIT_WRMSR:
10368 case VMX_EXIT_MWAIT:
10369 case VMX_EXIT_MONITOR:
10370 case VMX_EXIT_PAUSE:
10371 case VMX_EXIT_XDTR_ACCESS:
10372 case VMX_EXIT_TR_ACCESS:
10373 case VMX_EXIT_INVEPT:
10374 case VMX_EXIT_RDTSCP:
10375 case VMX_EXIT_INVVPID:
10376 case VMX_EXIT_WBINVD:
10377 case VMX_EXIT_XSETBV:
10378 case VMX_EXIT_RDRAND:
10379 case VMX_EXIT_INVPCID:
10380 case VMX_EXIT_VMFUNC:
10381 case VMX_EXIT_RDSEED:
10382 case VMX_EXIT_XSAVES:
10383 case VMX_EXIT_XRSTORS:
10384 {
10385 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10386 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10387 AssertRCReturn(rc2, rc2);
10388 if ( pMixedCtx->rip != pDbgState->uRipStart
10389 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10390 return VINF_EM_DBG_STEPPED;
10391 break;
10392 }
10393
10394 /* Errors and unexpected events: */
10395 case VMX_EXIT_INIT_SIGNAL:
10396 case VMX_EXIT_SIPI:
10397 case VMX_EXIT_IO_SMI:
10398 case VMX_EXIT_SMI:
10399 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10400 case VMX_EXIT_ERR_MSR_LOAD:
10401 case VMX_EXIT_ERR_MACHINE_CHECK:
10402 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10403 break;
10404
10405 default:
10406 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10407 break;
10408 }
10409 }
10410
10411 /*
10412 * Check for debugger event breakpoints and dtrace probes.
10413 */
10414 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10415 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10416 {
10417 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10418 if (rcStrict != VINF_SUCCESS)
10419 return rcStrict;
10420 }
10421
10422 /*
10423 * Normal processing.
10424 */
10425#ifdef HMVMX_USE_FUNCTION_TABLE
10426 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10427#else
10428 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10429#endif
10430}
10431
10432
10433/**
10434 * Single steps guest code using VT-x.
10435 *
10436 * @returns Strict VBox status code (i.e. informational status codes too).
10437 * @param pVM The cross context VM structure.
10438 * @param pVCpu The cross context virtual CPU structure.
10439 * @param pCtx Pointer to the guest-CPU context.
10440 *
10441 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10442 */
10443static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10444{
10445 VMXTRANSIENT VmxTransient;
10446 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10447
10448 /* Set HMCPU indicators. */
10449 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10450 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10451 pVCpu->hm.s.fDebugWantRdTscExit = false;
10452 pVCpu->hm.s.fUsingDebugLoop = true;
10453
10454 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10455 VMXRUNDBGSTATE DbgState;
10456 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10457 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10458
10459 /*
10460 * The loop.
10461 */
10462 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10463 for (uint32_t cLoops = 0; ; cLoops++)
10464 {
10465 Assert(!HMR0SuspendPending());
10466 HMVMX_ASSERT_CPU_SAFE();
10467 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10468
10469 /*
10470 * Preparatory work for running guest code, this may force us to return
10471 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10472 */
10473 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10474 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10475 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10476 if (rcStrict != VINF_SUCCESS)
10477 break;
10478
10479 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10480 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10481
10482 /*
10483 * Now we can run the guest code.
10484 */
10485 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10486
10487 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10488
10489 /*
10490 * Restore any residual host-state and save any bits shared between host
10491 * and guest into the guest-CPU state. Re-enables interrupts!
10492 */
10493 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10494
10495 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10496 if (RT_SUCCESS(rcRun))
10497 { /* very likely */ }
10498 else
10499 {
10500 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10501 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10502 return rcRun;
10503 }
10504
10505 /* Profile the VM-exit. */
10506 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10508 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10509 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10510 HMVMX_START_EXIT_DISPATCH_PROF();
10511
10512 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10513
10514 /*
10515 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10516 */
10517 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10518 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10519 if (rcStrict != VINF_SUCCESS)
10520 break;
10521 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10522 {
10523 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10524 rcStrict = VINF_EM_RAW_INTERRUPT;
10525 break;
10526 }
10527
10528 /*
10529 * Stepping: Did the RIP change, if so, consider it a single step.
10530 * Otherwise, make sure one of the TFs gets set.
10531 */
10532 if (fStepping)
10533 {
10534 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10535 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10536 AssertRCReturn(rc2, rc2);
10537 if ( pCtx->rip != DbgState.uRipStart
10538 || pCtx->cs.Sel != DbgState.uCsStart)
10539 {
10540 rcStrict = VINF_EM_DBG_STEPPED;
10541 break;
10542 }
10543 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10544 }
10545
10546 /*
10547 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10548 */
10549 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10550 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10551 }
10552
10553 /*
10554 * Clear the X86_EFL_TF if necessary.
10555 */
10556 if (pVCpu->hm.s.fClearTrapFlag)
10557 {
10558 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10559 AssertRCReturn(rc2, rc2);
10560 pVCpu->hm.s.fClearTrapFlag = false;
10561 pCtx->eflags.Bits.u1TF = 0;
10562 }
10563 /** @todo there seems to be issues with the resume flag when the monitor trap
10564 * flag is pending without being used. Seen early in bios init when
10565 * accessing APIC page in protected mode. */
10566
10567 /*
10568 * Restore VM-exit control settings as we may not reenter this function the
10569 * next time around.
10570 */
10571 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10572
10573 /* Restore HMCPU indicators. */
10574 pVCpu->hm.s.fUsingDebugLoop = false;
10575 pVCpu->hm.s.fDebugWantRdTscExit = false;
10576 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10577
10578 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10579 return rcStrict;
10580}
10581
10582
10583/** @} */
10584
10585
10586/**
10587 * Checks if any expensive dtrace probes are enabled and we should go to the
10588 * debug loop.
10589 *
10590 * @returns true if we should use debug loop, false if not.
10591 */
10592static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10593{
10594 /* It's probably faster to OR the raw 32-bit counter variables together.
10595 Since the variables are in an array and the probes are next to one
10596 another (more or less), we have good locality. So, better read
10597 eight-nine cache lines ever time and only have one conditional, than
10598 128+ conditionals, right? */
10599 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10600 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10601 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10602 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10603 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10604 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10605 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10606 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10607 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10608 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10609 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10610 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10611 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10612 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10613 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10614 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10615 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10616 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10617 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10618 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10619 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10620 ) != 0
10621 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10622 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10623 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10624 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10625 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10626 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10627 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10628 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10629 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10630 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10631 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10632 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10633 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10634 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10635 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10636 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10637 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10638 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10639 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10640 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10641 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10642 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10643 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10644 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10645 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10646 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10647 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10648 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10649 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10650 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10651 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10652 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10653 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10654 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10655 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10656 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10657 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10658 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10659 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10660 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10661 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10662 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10663 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10664 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10665 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10666 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10667 ) != 0
10668 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10669 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10670 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10671 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10672 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10673 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10674 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10675 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10676 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10677 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10678 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10679 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10680 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10681 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10682 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10683 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10684 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10685 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10686 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10687 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10688 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10689 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10690 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10691 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10692 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10693 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10694 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10695 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10696 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10697 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10698 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10699 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10700 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10701 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10702 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10703 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10704 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10705 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10706 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10707 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10708 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10709 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10710 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10711 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10712 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10713 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10714 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10715 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10716 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10717 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10718 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10719 ) != 0;
10720}
10721
10722
10723/**
10724 * Runs the guest code using VT-x.
10725 *
10726 * @returns Strict VBox status code (i.e. informational status codes too).
10727 * @param pVM The cross context VM structure.
10728 * @param pVCpu The cross context virtual CPU structure.
10729 * @param pCtx Pointer to the guest-CPU context.
10730 */
10731VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10732{
10733 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10734 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10735 HMVMX_ASSERT_PREEMPT_SAFE();
10736
10737 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10738
10739 VBOXSTRICTRC rcStrict;
10740 if ( !pVCpu->hm.s.fUseDebugLoop
10741 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10742 && !DBGFIsStepping(pVCpu)
10743 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10744 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10745 else
10746 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10747
10748 if (rcStrict == VERR_EM_INTERPRETER)
10749 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10750 else if (rcStrict == VINF_EM_RESET)
10751 rcStrict = VINF_EM_TRIPLE_FAULT;
10752
10753 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10754 if (RT_FAILURE(rc2))
10755 {
10756 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10757 rcStrict = rc2;
10758 }
10759 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10760 return rcStrict;
10761}
10762
10763
10764#ifndef HMVMX_USE_FUNCTION_TABLE
10765DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10766{
10767# ifdef DEBUG_ramshankar
10768# define RETURN_EXIT_CALL(a_CallExpr) \
10769 do { \
10770 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10771 VBOXSTRICTRC rcStrict = a_CallExpr; \
10772 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10773 return rcStrict; \
10774 } while (0)
10775# else
10776# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10777# endif
10778 switch (rcReason)
10779 {
10780 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10781 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10782 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10783 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10784 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10785 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10786 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10787 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10788 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10789 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10790 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10791 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10792 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10793 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10794 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10795 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10796 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10797 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10798 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10799 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10800 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10801 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10802 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10803 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10804 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10805 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10806 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10807 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10808 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10809 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10810 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10811 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10812 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10813 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10814
10815 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10816 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10817 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10818 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10819 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10820 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10821 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10822 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10823 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10824
10825 case VMX_EXIT_VMCLEAR:
10826 case VMX_EXIT_VMLAUNCH:
10827 case VMX_EXIT_VMPTRLD:
10828 case VMX_EXIT_VMPTRST:
10829 case VMX_EXIT_VMREAD:
10830 case VMX_EXIT_VMRESUME:
10831 case VMX_EXIT_VMWRITE:
10832 case VMX_EXIT_VMXOFF:
10833 case VMX_EXIT_VMXON:
10834 case VMX_EXIT_INVEPT:
10835 case VMX_EXIT_INVVPID:
10836 case VMX_EXIT_VMFUNC:
10837 case VMX_EXIT_XSAVES:
10838 case VMX_EXIT_XRSTORS:
10839 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10840 case VMX_EXIT_ENCLS:
10841 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10842 case VMX_EXIT_PML_FULL:
10843 default:
10844 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10845 }
10846#undef RETURN_EXIT_CALL
10847}
10848#endif /* !HMVMX_USE_FUNCTION_TABLE */
10849
10850
10851#ifdef VBOX_STRICT
10852/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10853# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10854 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10855
10856# define HMVMX_ASSERT_PREEMPT_CPUID() \
10857 do { \
10858 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10859 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10860 } while (0)
10861
10862# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10863 do { \
10864 AssertPtr(pVCpu); \
10865 AssertPtr(pMixedCtx); \
10866 AssertPtr(pVmxTransient); \
10867 Assert(pVmxTransient->fVMEntryFailed == false); \
10868 Assert(ASMIntAreEnabled()); \
10869 HMVMX_ASSERT_PREEMPT_SAFE(); \
10870 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10871 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10872 HMVMX_ASSERT_PREEMPT_SAFE(); \
10873 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10874 HMVMX_ASSERT_PREEMPT_CPUID(); \
10875 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10876 } while (0)
10877
10878# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10879 do { \
10880 Log4Func(("\n")); \
10881 } while (0)
10882#else /* nonstrict builds: */
10883# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10884 do { \
10885 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10886 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10887 } while (0)
10888# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10889#endif
10890
10891
10892/**
10893 * Advances the guest RIP by the specified number of bytes.
10894 *
10895 * @param pVCpu The cross context virtual CPU structure.
10896 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10897 * out-of-sync. Make sure to update the required fields
10898 * before using them.
10899 * @param cbInstr Number of bytes to advance the RIP by.
10900 *
10901 * @remarks No-long-jump zone!!!
10902 */
10903DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10904{
10905 /* Advance the RIP. */
10906 pMixedCtx->rip += cbInstr;
10907 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10908
10909 /* Update interrupt inhibition. */
10910 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10911 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10912 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10913}
10914
10915
10916/**
10917 * Advances the guest RIP after reading it from the VMCS.
10918 *
10919 * @returns VBox status code, no informational status codes.
10920 * @param pVCpu The cross context virtual CPU structure.
10921 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10922 * out-of-sync. Make sure to update the required fields
10923 * before using them.
10924 * @param pVmxTransient Pointer to the VMX transient structure.
10925 *
10926 * @remarks No-long-jump zone!!!
10927 */
10928static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10929{
10930 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10931 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10932 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10933 AssertRCReturn(rc, rc);
10934
10935 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10936
10937 /*
10938 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10939 * pending debug exception field as it takes care of priority of events.
10940 *
10941 * See Intel spec. 32.2.1 "Debug Exceptions".
10942 */
10943 if ( !pVCpu->hm.s.fSingleInstruction
10944 && pMixedCtx->eflags.Bits.u1TF)
10945 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10946
10947 return VINF_SUCCESS;
10948}
10949
10950
10951/**
10952 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10953 * and update error record fields accordingly.
10954 *
10955 * @return VMX_IGS_* return codes.
10956 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10957 * wrong with the guest state.
10958 *
10959 * @param pVM The cross context VM structure.
10960 * @param pVCpu The cross context virtual CPU structure.
10961 * @param pCtx Pointer to the guest-CPU state.
10962 *
10963 * @remarks This function assumes our cache of the VMCS controls
10964 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10965 */
10966static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10967{
10968#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10969#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10970 uError = (err); \
10971 break; \
10972 } else do { } while (0)
10973
10974 int rc;
10975 uint32_t uError = VMX_IGS_ERROR;
10976 uint32_t u32Val;
10977 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10978
10979 do
10980 {
10981 /*
10982 * CR0.
10983 */
10984 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10985 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10986 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10987 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10988 if (fUnrestrictedGuest)
10989 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10990
10991 uint32_t u32GuestCR0;
10992 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10993 AssertRCBreak(rc);
10994 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10995 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10996 if ( !fUnrestrictedGuest
10997 && (u32GuestCR0 & X86_CR0_PG)
10998 && !(u32GuestCR0 & X86_CR0_PE))
10999 {
11000 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
11001 }
11002
11003 /*
11004 * CR4.
11005 */
11006 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11007 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11008
11009 uint32_t u32GuestCR4;
11010 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11011 AssertRCBreak(rc);
11012 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11013 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11014
11015 /*
11016 * IA32_DEBUGCTL MSR.
11017 */
11018 uint64_t u64Val;
11019 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11020 AssertRCBreak(rc);
11021 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11022 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11023 {
11024 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11025 }
11026 uint64_t u64DebugCtlMsr = u64Val;
11027
11028#ifdef VBOX_STRICT
11029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11030 AssertRCBreak(rc);
11031 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11032#endif
11033 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11034
11035 /*
11036 * RIP and RFLAGS.
11037 */
11038 uint32_t u32Eflags;
11039#if HC_ARCH_BITS == 64
11040 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11041 AssertRCBreak(rc);
11042 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11043 if ( !fLongModeGuest
11044 || !pCtx->cs.Attr.n.u1Long)
11045 {
11046 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11047 }
11048 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11049 * must be identical if the "IA-32e mode guest" VM-entry
11050 * control is 1 and CS.L is 1. No check applies if the
11051 * CPU supports 64 linear-address bits. */
11052
11053 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11054 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11055 AssertRCBreak(rc);
11056 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11057 VMX_IGS_RFLAGS_RESERVED);
11058 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11059 u32Eflags = u64Val;
11060#else
11061 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11062 AssertRCBreak(rc);
11063 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11064 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11065#endif
11066
11067 if ( fLongModeGuest
11068 || ( fUnrestrictedGuest
11069 && !(u32GuestCR0 & X86_CR0_PE)))
11070 {
11071 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11072 }
11073
11074 uint32_t u32EntryInfo;
11075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11076 AssertRCBreak(rc);
11077 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11078 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11079 {
11080 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11081 }
11082
11083 /*
11084 * 64-bit checks.
11085 */
11086#if HC_ARCH_BITS == 64
11087 if (fLongModeGuest)
11088 {
11089 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11090 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11091 }
11092
11093 if ( !fLongModeGuest
11094 && (u32GuestCR4 & X86_CR4_PCIDE))
11095 {
11096 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11097 }
11098
11099 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11100 * 51:32 beyond the processor's physical-address width are 0. */
11101
11102 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11103 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11104 {
11105 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11106 }
11107
11108 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11109 AssertRCBreak(rc);
11110 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11111
11112 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11113 AssertRCBreak(rc);
11114 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11115#endif
11116
11117 /*
11118 * PERF_GLOBAL MSR.
11119 */
11120 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11121 {
11122 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11123 AssertRCBreak(rc);
11124 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11125 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11126 }
11127
11128 /*
11129 * PAT MSR.
11130 */
11131 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11132 {
11133 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11134 AssertRCBreak(rc);
11135 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11136 for (unsigned i = 0; i < 8; i++)
11137 {
11138 uint8_t u8Val = (u64Val & 0xff);
11139 if ( u8Val != 0 /* UC */
11140 && u8Val != 1 /* WC */
11141 && u8Val != 4 /* WT */
11142 && u8Val != 5 /* WP */
11143 && u8Val != 6 /* WB */
11144 && u8Val != 7 /* UC- */)
11145 {
11146 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11147 }
11148 u64Val >>= 8;
11149 }
11150 }
11151
11152 /*
11153 * EFER MSR.
11154 */
11155 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11156 {
11157 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11158 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11159 AssertRCBreak(rc);
11160 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11161 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11162 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11163 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11164 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11165 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11166 || !(u32GuestCR0 & X86_CR0_PG)
11167 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11168 VMX_IGS_EFER_LMA_LME_MISMATCH);
11169 }
11170
11171 /*
11172 * Segment registers.
11173 */
11174 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11175 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11176 if (!(u32Eflags & X86_EFL_VM))
11177 {
11178 /* CS */
11179 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11180 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11181 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11182 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11183 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11184 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11185 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11186 /* CS cannot be loaded with NULL in protected mode. */
11187 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11188 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11189 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11190 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11191 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11192 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11193 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11194 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11195 else
11196 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11197
11198 /* SS */
11199 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11200 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11201 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11202 if ( !(pCtx->cr0 & X86_CR0_PE)
11203 || pCtx->cs.Attr.n.u4Type == 3)
11204 {
11205 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11206 }
11207 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11208 {
11209 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11210 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11211 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11212 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11213 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11214 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11215 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11216 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11217 }
11218
11219 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11220 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11221 {
11222 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11223 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11224 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11225 || pCtx->ds.Attr.n.u4Type > 11
11226 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11227 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11228 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11229 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11230 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11231 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11232 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11233 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11234 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11235 }
11236 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11237 {
11238 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11239 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11240 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11241 || pCtx->es.Attr.n.u4Type > 11
11242 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11243 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11244 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11245 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11246 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11247 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11248 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11249 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11250 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11251 }
11252 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11253 {
11254 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11255 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11256 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11257 || pCtx->fs.Attr.n.u4Type > 11
11258 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11259 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11260 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11261 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11262 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11263 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11264 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11265 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11266 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11267 }
11268 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11269 {
11270 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11271 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11272 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11273 || pCtx->gs.Attr.n.u4Type > 11
11274 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11275 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11276 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11277 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11278 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11279 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11280 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11281 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11282 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11283 }
11284 /* 64-bit capable CPUs. */
11285#if HC_ARCH_BITS == 64
11286 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11287 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11288 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11289 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11290 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11291 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11292 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11293 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11294 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11295 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11296 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11297#endif
11298 }
11299 else
11300 {
11301 /* V86 mode checks. */
11302 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11303 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11304 {
11305 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11306 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11307 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11308 }
11309 else
11310 {
11311 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11312 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11313 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11314 }
11315
11316 /* CS */
11317 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11318 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11319 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11320 /* SS */
11321 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11322 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11323 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11324 /* DS */
11325 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11326 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11327 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11328 /* ES */
11329 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11330 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11331 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11332 /* FS */
11333 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11334 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11335 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11336 /* GS */
11337 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11338 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11339 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11340 /* 64-bit capable CPUs. */
11341#if HC_ARCH_BITS == 64
11342 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11343 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11344 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11345 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11346 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11347 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11348 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11349 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11350 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11351 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11352 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11353#endif
11354 }
11355
11356 /*
11357 * TR.
11358 */
11359 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11360 /* 64-bit capable CPUs. */
11361#if HC_ARCH_BITS == 64
11362 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11363#endif
11364 if (fLongModeGuest)
11365 {
11366 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11367 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11368 }
11369 else
11370 {
11371 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11372 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11373 VMX_IGS_TR_ATTR_TYPE_INVALID);
11374 }
11375 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11376 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11377 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11378 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11379 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11380 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11381 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11382 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11383
11384 /*
11385 * GDTR and IDTR.
11386 */
11387#if HC_ARCH_BITS == 64
11388 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11389 AssertRCBreak(rc);
11390 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11391
11392 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11393 AssertRCBreak(rc);
11394 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11395#endif
11396
11397 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11398 AssertRCBreak(rc);
11399 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11400
11401 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11402 AssertRCBreak(rc);
11403 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11404
11405 /*
11406 * Guest Non-Register State.
11407 */
11408 /* Activity State. */
11409 uint32_t u32ActivityState;
11410 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11411 AssertRCBreak(rc);
11412 HMVMX_CHECK_BREAK( !u32ActivityState
11413 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11414 VMX_IGS_ACTIVITY_STATE_INVALID);
11415 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11416 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11417 uint32_t u32IntrState;
11418 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11419 AssertRCBreak(rc);
11420 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11421 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11422 {
11423 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11424 }
11425
11426 /** @todo Activity state and injecting interrupts. Left as a todo since we
11427 * currently don't use activity states but ACTIVE. */
11428
11429 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11430 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11431
11432 /* Guest interruptibility-state. */
11433 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11434 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11435 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11436 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11437 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11438 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11439 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11440 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11441 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11442 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11443 {
11444 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11445 {
11446 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11447 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11448 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11449 }
11450 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11451 {
11452 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11453 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11454 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11455 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11456 }
11457 }
11458 /** @todo Assumes the processor is not in SMM. */
11459 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11460 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11461 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11462 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11463 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11464 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11465 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11466 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11467 {
11468 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11469 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11470 }
11471
11472 /* Pending debug exceptions. */
11473#if HC_ARCH_BITS == 64
11474 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11475 AssertRCBreak(rc);
11476 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11477 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11478 u32Val = u64Val; /* For pending debug exceptions checks below. */
11479#else
11480 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11481 AssertRCBreak(rc);
11482 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11483 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11484#endif
11485
11486 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11487 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11488 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11489 {
11490 if ( (u32Eflags & X86_EFL_TF)
11491 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11492 {
11493 /* Bit 14 is PendingDebug.BS. */
11494 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11495 }
11496 if ( !(u32Eflags & X86_EFL_TF)
11497 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11498 {
11499 /* Bit 14 is PendingDebug.BS. */
11500 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11501 }
11502 }
11503
11504 /* VMCS link pointer. */
11505 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11506 AssertRCBreak(rc);
11507 if (u64Val != UINT64_C(0xffffffffffffffff))
11508 {
11509 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11510 /** @todo Bits beyond the processor's physical-address width MBZ. */
11511 /** @todo 32-bit located in memory referenced by value of this field (as a
11512 * physical address) must contain the processor's VMCS revision ID. */
11513 /** @todo SMM checks. */
11514 }
11515
11516 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11517 * not using Nested Paging? */
11518 if ( pVM->hm.s.fNestedPaging
11519 && !fLongModeGuest
11520 && CPUMIsGuestInPAEModeEx(pCtx))
11521 {
11522 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11523 AssertRCBreak(rc);
11524 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11525
11526 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11527 AssertRCBreak(rc);
11528 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11529
11530 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11531 AssertRCBreak(rc);
11532 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11533
11534 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11535 AssertRCBreak(rc);
11536 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11537 }
11538
11539 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11540 if (uError == VMX_IGS_ERROR)
11541 uError = VMX_IGS_REASON_NOT_FOUND;
11542 } while (0);
11543
11544 pVCpu->hm.s.u32HMError = uError;
11545 return uError;
11546
11547#undef HMVMX_ERROR_BREAK
11548#undef HMVMX_CHECK_BREAK
11549}
11550
11551/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11552/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11553/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11554
11555/** @name VM-exit handlers.
11556 * @{
11557 */
11558
11559/**
11560 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11561 */
11562HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11563{
11564 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11565 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11566 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11567 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11568 return VINF_SUCCESS;
11569 return VINF_EM_RAW_INTERRUPT;
11570}
11571
11572
11573/**
11574 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11575 */
11576HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11577{
11578 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11579 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11580
11581 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11582 AssertRCReturn(rc, rc);
11583
11584 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11585 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11586 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11587 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11588
11589 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11590 {
11591 /*
11592 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11593 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11594 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11595 *
11596 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11597 */
11598 VMXDispatchHostNmi();
11599 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11600 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11601 return VINF_SUCCESS;
11602 }
11603
11604 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11605 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11606 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11607 { /* likely */ }
11608 else
11609 {
11610 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11611 rcStrictRc1 = VINF_SUCCESS;
11612 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11613 return rcStrictRc1;
11614 }
11615
11616 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11617 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11618 switch (uIntType)
11619 {
11620 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11621 Assert(uVector == X86_XCPT_DB);
11622 RT_FALL_THRU();
11623 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11624 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11625 RT_FALL_THRU();
11626 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11627 {
11628 /*
11629 * If there's any exception caused as a result of event injection, the resulting
11630 * secondary/final execption will be pending, we shall continue guest execution
11631 * after injecting the event. The page-fault case is complicated and we manually
11632 * handle any currently pending event in hmR0VmxExitXcptPF.
11633 */
11634 if (!pVCpu->hm.s.Event.fPending)
11635 { /* likely */ }
11636 else if (uVector != X86_XCPT_PF)
11637 {
11638 rc = VINF_SUCCESS;
11639 break;
11640 }
11641
11642 switch (uVector)
11643 {
11644 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11645 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11646 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11647 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11648 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11649 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11650
11651 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11652 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11653 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11654 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11655 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11656 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11657 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11658 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11659 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11660 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11661 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11662 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11663 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11664 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11665 default:
11666 {
11667 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11668 AssertRCReturn(rc, rc);
11669
11670 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11671 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11672 {
11673 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11674 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11675 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11676
11677 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11678 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11679 AssertRCReturn(rc, rc);
11680 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11681 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11682 0 /* GCPtrFaultAddress */);
11683 AssertRCReturn(rc, rc);
11684 }
11685 else
11686 {
11687 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11688 pVCpu->hm.s.u32HMError = uVector;
11689 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11690 }
11691 break;
11692 }
11693 }
11694 break;
11695 }
11696
11697 default:
11698 {
11699 pVCpu->hm.s.u32HMError = uExitIntInfo;
11700 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11701 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11702 break;
11703 }
11704 }
11705 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11706 return rc;
11707}
11708
11709
11710/**
11711 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11712 */
11713HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11714{
11715 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11716
11717 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11718 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11719
11720 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11721 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11722 return VINF_SUCCESS;
11723}
11724
11725
11726/**
11727 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11728 */
11729HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11730{
11731 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11732 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11733 {
11734 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11735 HMVMX_RETURN_UNEXPECTED_EXIT();
11736 }
11737
11738 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11739
11740 /*
11741 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11742 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11743 */
11744 uint32_t uIntrState = 0;
11745 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11746 AssertRCReturn(rc, rc);
11747
11748 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11749 if ( fBlockSti
11750 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11751 {
11752 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11753 }
11754
11755 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11756 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11757
11758 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11759 return VINF_SUCCESS;
11760}
11761
11762
11763/**
11764 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11765 */
11766HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11767{
11768 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11769 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11770 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11771}
11772
11773
11774/**
11775 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11776 */
11777HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11778{
11779 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11780 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11781 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11782}
11783
11784
11785/**
11786 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11787 */
11788HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11789{
11790 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11791 PVM pVM = pVCpu->CTX_SUFF(pVM);
11792 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11793 if (RT_LIKELY(rc == VINF_SUCCESS))
11794 {
11795 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11796 Assert(pVmxTransient->cbInstr == 2);
11797 }
11798 else
11799 {
11800 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11801 rc = VERR_EM_INTERPRETER;
11802 }
11803 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11804 return rc;
11805}
11806
11807
11808/**
11809 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11810 */
11811HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11812{
11813 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11814 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11815 AssertRCReturn(rc, rc);
11816
11817 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11818 return VINF_EM_RAW_EMULATE_INSTR;
11819
11820 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11821 HMVMX_RETURN_UNEXPECTED_EXIT();
11822}
11823
11824
11825/**
11826 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11827 */
11828HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11829{
11830 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11831 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
11832 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11833 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11834 AssertRCReturn(rc, rc);
11835 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11836 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11837 {
11838 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11839 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11840 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11841 }
11842 else if (rcStrict == VINF_EM_RESCHEDULE)
11843 rcStrict = VINF_SUCCESS;
11844 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11845 return rcStrict;
11846}
11847
11848
11849/**
11850 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11851 */
11852HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11853{
11854 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11855 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx); /* Needed for CPL < 0 only, really. */
11856 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
11857 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11858 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11859 AssertRCReturn(rc, rc);
11860 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11861 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11862 {
11863 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11864 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11865 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11866 }
11867 else if (rcStrict == VINF_EM_RESCHEDULE)
11868 rcStrict = VINF_SUCCESS;
11869 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtscp);
11870 return rcStrict;
11871}
11872
11873
11874/**
11875 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11876 */
11877HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11878{
11879 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11880 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11881 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11882 AssertRCReturn(rc, rc);
11883
11884 PVM pVM = pVCpu->CTX_SUFF(pVM);
11885 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11886 if (RT_LIKELY(rc == VINF_SUCCESS))
11887 {
11888 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11889 Assert(pVmxTransient->cbInstr == 2);
11890 }
11891 else
11892 {
11893 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11894 rc = VERR_EM_INTERPRETER;
11895 }
11896 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11897 return rc;
11898}
11899
11900
11901/**
11902 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11903 */
11904HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11905{
11906 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11907 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11908
11909 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11910 if (EMAreHypercallInstructionsEnabled(pVCpu))
11911 {
11912#if 0
11913 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11914#else
11915 /* Aggressive state sync. for now. */
11916 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11917 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11918 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11919 AssertRCReturn(rc, rc);
11920#endif
11921
11922 /* Perform the hypercall. */
11923 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11924 if (rcStrict == VINF_SUCCESS)
11925 {
11926 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11927 AssertRCReturn(rc, rc);
11928 }
11929 else
11930 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11931 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11932 || RT_FAILURE(rcStrict));
11933
11934 /* If the hypercall changes anything other than guest's general-purpose registers,
11935 we would need to reload the guest changed bits here before VM-entry. */
11936 }
11937 else
11938 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11939
11940 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11941 if (RT_FAILURE(rcStrict))
11942 {
11943 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11944 rcStrict = VINF_SUCCESS;
11945 }
11946
11947 return rcStrict;
11948}
11949
11950
11951/**
11952 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11953 */
11954HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11955{
11956 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11957 PVM pVM = pVCpu->CTX_SUFF(pVM);
11958 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11959
11960 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11961 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11962 AssertRCReturn(rc, rc);
11963
11964 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11965 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11966 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11967 else
11968 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11969 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11971 return rcStrict;
11972}
11973
11974
11975/**
11976 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11977 */
11978HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11979{
11980 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11981 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11982 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11983 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11984 AssertRCReturn(rc, rc);
11985
11986 PVM pVM = pVCpu->CTX_SUFF(pVM);
11987 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11988 if (RT_LIKELY(rc == VINF_SUCCESS))
11989 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11990 else
11991 {
11992 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11993 rc = VERR_EM_INTERPRETER;
11994 }
11995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11996 return rc;
11997}
11998
11999
12000/**
12001 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12002 */
12003HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12004{
12005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12006 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12007 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12008 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12009 AssertRCReturn(rc, rc);
12010
12011 PVM pVM = pVCpu->CTX_SUFF(pVM);
12012 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12013 rc = VBOXSTRICTRC_VAL(rc2);
12014 if (RT_LIKELY( rc == VINF_SUCCESS
12015 || rc == VINF_EM_HALT))
12016 {
12017 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12018 AssertRCReturn(rc3, rc3);
12019
12020 if ( rc == VINF_EM_HALT
12021 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12022 {
12023 rc = VINF_SUCCESS;
12024 }
12025 }
12026 else
12027 {
12028 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12029 rc = VERR_EM_INTERPRETER;
12030 }
12031 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
12032 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12033 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12034 return rc;
12035}
12036
12037
12038/**
12039 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12040 */
12041HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12042{
12043 /*
12044 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12045 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12046 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12047 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12048 */
12049 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12050 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12051 HMVMX_RETURN_UNEXPECTED_EXIT();
12052}
12053
12054
12055/**
12056 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12057 */
12058HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12059{
12060 /*
12061 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12062 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12063 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12064 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12065 */
12066 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12067 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12068 HMVMX_RETURN_UNEXPECTED_EXIT();
12069}
12070
12071
12072/**
12073 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12074 */
12075HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12076{
12077 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12078 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12079 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12080 HMVMX_RETURN_UNEXPECTED_EXIT();
12081}
12082
12083
12084/**
12085 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12086 */
12087HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12088{
12089 /*
12090 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12091 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12092 * See Intel spec. 25.3 "Other Causes of VM-exits".
12093 */
12094 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12095 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12096 HMVMX_RETURN_UNEXPECTED_EXIT();
12097}
12098
12099
12100/**
12101 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12102 * VM-exit.
12103 */
12104HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12105{
12106 /*
12107 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12108 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12109 *
12110 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12111 * See Intel spec. "23.8 Restrictions on VMX operation".
12112 */
12113 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12114 return VINF_SUCCESS;
12115}
12116
12117
12118/**
12119 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12120 * VM-exit.
12121 */
12122HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12123{
12124 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12125 return VINF_EM_RESET;
12126}
12127
12128
12129/**
12130 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12131 */
12132HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12133{
12134 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12135 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12136
12137 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12138 AssertRCReturn(rc, rc);
12139
12140 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12141 rc = VINF_SUCCESS;
12142 else
12143 rc = VINF_EM_HALT;
12144
12145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12146 if (rc != VINF_SUCCESS)
12147 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12148 return rc;
12149}
12150
12151
12152/**
12153 * VM-exit handler for instructions that result in a \#UD exception delivered to
12154 * the guest.
12155 */
12156HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12157{
12158 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12159 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12160 return VINF_SUCCESS;
12161}
12162
12163
12164/**
12165 * VM-exit handler for expiry of the VMX preemption timer.
12166 */
12167HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12168{
12169 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12170
12171 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12172 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12173
12174 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12175 PVM pVM = pVCpu->CTX_SUFF(pVM);
12176 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12177 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12178 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12179}
12180
12181
12182/**
12183 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12184 */
12185HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12186{
12187 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12188
12189 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12190 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12191 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12192 AssertRCReturn(rc, rc);
12193
12194 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12195 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12196
12197 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12198
12199 return rcStrict;
12200}
12201
12202
12203/**
12204 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12205 */
12206HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12207{
12208 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12209 /** @todo Use VM-exit instruction information. */
12210 return VERR_EM_INTERPRETER;
12211}
12212
12213
12214/**
12215 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12216 * Error VM-exit.
12217 */
12218HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12219{
12220 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12221 AssertRCReturn(rc, rc);
12222
12223 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12224 AssertRCReturn(rc, rc);
12225
12226 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12227 NOREF(uInvalidReason);
12228
12229#ifdef VBOX_STRICT
12230 uint32_t uIntrState;
12231 RTHCUINTREG uHCReg;
12232 uint64_t u64Val;
12233 uint32_t u32Val;
12234
12235 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12236 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12237 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12238 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12239 AssertRCReturn(rc, rc);
12240
12241 Log4(("uInvalidReason %u\n", uInvalidReason));
12242 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12243 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12244 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12245 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12246
12247 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12248 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12249 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12250 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12251 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12252 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12253 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12254 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12255 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12256 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12257 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12258 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12259#else
12260 NOREF(pVmxTransient);
12261#endif
12262
12263 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12264 return VERR_VMX_INVALID_GUEST_STATE;
12265}
12266
12267
12268/**
12269 * VM-exit handler for VM-entry failure due to an MSR-load
12270 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12271 */
12272HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12273{
12274 NOREF(pVmxTransient);
12275 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12276 HMVMX_RETURN_UNEXPECTED_EXIT();
12277}
12278
12279
12280/**
12281 * VM-exit handler for VM-entry failure due to a machine-check event
12282 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12283 */
12284HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12285{
12286 NOREF(pVmxTransient);
12287 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12288 HMVMX_RETURN_UNEXPECTED_EXIT();
12289}
12290
12291
12292/**
12293 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12294 * theory.
12295 */
12296HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12297{
12298 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12299 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12300 return VERR_VMX_UNDEFINED_EXIT_CODE;
12301}
12302
12303
12304/**
12305 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12306 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12307 * Conditional VM-exit.
12308 */
12309HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12310{
12311 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12312
12313 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12314 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12315 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12316 return VERR_EM_INTERPRETER;
12317 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12318 HMVMX_RETURN_UNEXPECTED_EXIT();
12319}
12320
12321
12322/**
12323 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12324 */
12325HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12326{
12327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12328
12329 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12330 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12331 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12332 return VERR_EM_INTERPRETER;
12333 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12334 HMVMX_RETURN_UNEXPECTED_EXIT();
12335}
12336
12337
12338/**
12339 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12340 */
12341HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12342{
12343 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12344
12345 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12346 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12347 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12348 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12349 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12350 {
12351 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12352 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12353 }
12354 AssertRCReturn(rc, rc);
12355 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12356
12357#ifdef VBOX_STRICT
12358 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12359 {
12360 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12361 && pMixedCtx->ecx != MSR_K6_EFER)
12362 {
12363 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12364 pMixedCtx->ecx));
12365 HMVMX_RETURN_UNEXPECTED_EXIT();
12366 }
12367 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12368 {
12369 VMXMSREXITREAD enmRead;
12370 VMXMSREXITWRITE enmWrite;
12371 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12372 AssertRCReturn(rc2, rc2);
12373 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12374 {
12375 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12376 HMVMX_RETURN_UNEXPECTED_EXIT();
12377 }
12378 }
12379 }
12380#endif
12381
12382 PVM pVM = pVCpu->CTX_SUFF(pVM);
12383 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12384 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12385 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12387 if (RT_SUCCESS(rc))
12388 {
12389 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12390 Assert(pVmxTransient->cbInstr == 2);
12391 }
12392 return rc;
12393}
12394
12395
12396/**
12397 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12398 */
12399HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12400{
12401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12402 PVM pVM = pVCpu->CTX_SUFF(pVM);
12403 int rc = VINF_SUCCESS;
12404
12405 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12406 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12407 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12408 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12409 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12410 {
12411 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12412 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12413 }
12414 AssertRCReturn(rc, rc);
12415 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12416
12417 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12418 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12420
12421 if (RT_SUCCESS(rc))
12422 {
12423 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12424
12425 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12426 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12427 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12428 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12429 {
12430 /*
12431 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12432 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12433 * EMInterpretWrmsr() changes it.
12434 */
12435 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12436 }
12437 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12438 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12439 else if (pMixedCtx->ecx == MSR_K6_EFER)
12440 {
12441 /*
12442 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12443 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12444 * the other bits as well, SCE and NXE. See @bugref{7368}.
12445 */
12446 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12447 }
12448
12449 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12450 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12451 {
12452 switch (pMixedCtx->ecx)
12453 {
12454 /*
12455 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12456 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12457 */
12458 case MSR_IA32_SYSENTER_CS:
12459 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12460 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12461 break;
12462 case MSR_IA32_SYSENTER_EIP:
12463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12464 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12465 break;
12466 case MSR_IA32_SYSENTER_ESP:
12467 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12468 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12469 break;
12470 case MSR_K8_FS_BASE: RT_FALL_THRU();
12471 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12472 case MSR_K6_EFER: /* already handled above */ break;
12473 default:
12474 {
12475 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12476 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12477 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12478 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12479 break;
12480 }
12481 }
12482 }
12483#ifdef VBOX_STRICT
12484 else
12485 {
12486 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12487 switch (pMixedCtx->ecx)
12488 {
12489 case MSR_IA32_SYSENTER_CS:
12490 case MSR_IA32_SYSENTER_EIP:
12491 case MSR_IA32_SYSENTER_ESP:
12492 case MSR_K8_FS_BASE:
12493 case MSR_K8_GS_BASE:
12494 {
12495 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12496 HMVMX_RETURN_UNEXPECTED_EXIT();
12497 }
12498
12499 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12500 default:
12501 {
12502 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12503 {
12504 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12505 if (pMixedCtx->ecx != MSR_K6_EFER)
12506 {
12507 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12508 pMixedCtx->ecx));
12509 HMVMX_RETURN_UNEXPECTED_EXIT();
12510 }
12511 }
12512
12513 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12514 {
12515 VMXMSREXITREAD enmRead;
12516 VMXMSREXITWRITE enmWrite;
12517 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12518 AssertRCReturn(rc2, rc2);
12519 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12520 {
12521 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12522 HMVMX_RETURN_UNEXPECTED_EXIT();
12523 }
12524 }
12525 break;
12526 }
12527 }
12528 }
12529#endif /* VBOX_STRICT */
12530 }
12531 return rc;
12532}
12533
12534
12535/**
12536 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12537 */
12538HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12539{
12540 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12541
12542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12543 return VINF_EM_RAW_INTERRUPT;
12544}
12545
12546
12547/**
12548 * VM-exit handler for when the TPR value is lowered below the specified
12549 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12550 */
12551HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12552{
12553 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12554 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12555
12556 /*
12557 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12558 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12559 */
12560 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12561 return VINF_SUCCESS;
12562}
12563
12564
12565/**
12566 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12567 * VM-exit.
12568 *
12569 * @retval VINF_SUCCESS when guest execution can continue.
12570 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12571 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12572 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12573 * interpreter.
12574 */
12575HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12576{
12577 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12578 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12579 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12580 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12581 AssertRCReturn(rc, rc);
12582
12583 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12584 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12585 PVM pVM = pVCpu->CTX_SUFF(pVM);
12586 VBOXSTRICTRC rcStrict;
12587 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12588 switch (uAccessType)
12589 {
12590 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12591 {
12592 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12593 AssertRCReturn(rc, rc);
12594
12595 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12596 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12597 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12598 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12599 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12600 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12601 {
12602 case 0: /* CR0 */
12603 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12604 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12605 break;
12606 case 2: /* CR2 */
12607 /* Nothing to do here, CR2 it's not part of the VMCS. */
12608 break;
12609 case 3: /* CR3 */
12610 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12611 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12612 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12613 break;
12614 case 4: /* CR4 */
12615 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12616 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12617 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12618 break;
12619 case 8: /* CR8 */
12620 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12621 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12622 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12623 break;
12624 default:
12625 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12626 break;
12627 }
12628
12629 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12630 break;
12631 }
12632
12633 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12634 {
12635 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12636 AssertRCReturn(rc, rc);
12637
12638 Assert( !pVM->hm.s.fNestedPaging
12639 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12640 || pVCpu->hm.s.fUsingDebugLoop
12641 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12642
12643 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12644 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12645 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12646
12647 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12648 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12649 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12650 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12651 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12652 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12653 VBOXSTRICTRC_VAL(rcStrict)));
12654 if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12655 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12656 break;
12657 }
12658
12659 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12660 {
12661 AssertRCReturn(rc, rc);
12662 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12663 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12664 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12666 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12667 break;
12668 }
12669
12670 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12671 {
12672 AssertRCReturn(rc, rc);
12673 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12674 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12675 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12676 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12677 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12678 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12679 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12680 break;
12681 }
12682
12683 default:
12684 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12685 VERR_VMX_UNEXPECTED_EXCEPTION);
12686 }
12687
12688 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12689 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12690 NOREF(pVM);
12691 return rcStrict;
12692}
12693
12694
12695/**
12696 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12697 * VM-exit.
12698 */
12699HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12700{
12701 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12702 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12703
12704 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12705 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12706 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12707 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12708 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12709 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12710 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12711 AssertRCReturn(rc, rc);
12712
12713 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12714 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12715 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12716 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12717 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12718 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12719 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12720 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12721 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12722
12723 /* I/O operation lookup arrays. */
12724 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12725 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12726
12727 VBOXSTRICTRC rcStrict;
12728 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12729 uint32_t const cbInstr = pVmxTransient->cbInstr;
12730 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12731 PVM pVM = pVCpu->CTX_SUFF(pVM);
12732 if (fIOString)
12733 {
12734 /*
12735 * INS/OUTS - I/O String instruction.
12736 *
12737 * Use instruction-information if available, otherwise fall back on
12738 * interpreting the instruction.
12739 */
12740 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12741 fIOWrite ? 'w' : 'r'));
12742 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12743 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12744 {
12745 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12746 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12747 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12748 AssertRCReturn(rc2, rc2);
12749 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12750 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12751 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12752 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12753 if (fIOWrite)
12754 {
12755 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12756 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12757 }
12758 else
12759 {
12760 /*
12761 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12762 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12763 * See Intel Instruction spec. for "INS".
12764 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12765 */
12766 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12767 }
12768 }
12769 else
12770 {
12771 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12772 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12773 AssertRCReturn(rc2, rc2);
12774 rcStrict = IEMExecOne(pVCpu);
12775 }
12776 /** @todo IEM needs to be setting these flags somehow. */
12777 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12778 fUpdateRipAlready = true;
12779 }
12780 else
12781 {
12782 /*
12783 * IN/OUT - I/O instruction.
12784 */
12785 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12786 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12787 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12788 if (fIOWrite)
12789 {
12790 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12791 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12792 }
12793 else
12794 {
12795 uint32_t u32Result = 0;
12796 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12797 if (IOM_SUCCESS(rcStrict))
12798 {
12799 /* Save result of I/O IN instr. in AL/AX/EAX. */
12800 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12801 }
12802 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12803 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12804 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12805 }
12806 }
12807
12808 if (IOM_SUCCESS(rcStrict))
12809 {
12810 if (!fUpdateRipAlready)
12811 {
12812 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12813 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12814 }
12815
12816 /*
12817 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12818 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12819 */
12820 if (fIOString)
12821 {
12822 /** @todo Single-step for INS/OUTS with REP prefix? */
12823 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12824 }
12825 else if ( !fDbgStepping
12826 && fGstStepping)
12827 {
12828 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12829 }
12830
12831 /*
12832 * If any I/O breakpoints are armed, we need to check if one triggered
12833 * and take appropriate action.
12834 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12835 */
12836 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12837 AssertRCReturn(rc2, rc2);
12838
12839 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12840 * execution engines about whether hyper BPs and such are pending. */
12841 uint32_t const uDr7 = pMixedCtx->dr[7];
12842 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12843 && X86_DR7_ANY_RW_IO(uDr7)
12844 && (pMixedCtx->cr4 & X86_CR4_DE))
12845 || DBGFBpIsHwIoArmed(pVM)))
12846 {
12847 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12848
12849 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12850 VMMRZCallRing3Disable(pVCpu);
12851 HM_DISABLE_PREEMPT();
12852
12853 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12854
12855 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12856 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12857 {
12858 /* Raise #DB. */
12859 if (fIsGuestDbgActive)
12860 ASMSetDR6(pMixedCtx->dr[6]);
12861 if (pMixedCtx->dr[7] != uDr7)
12862 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12863
12864 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12865 }
12866 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12867 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12868 else if ( rcStrict2 != VINF_SUCCESS
12869 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12870 rcStrict = rcStrict2;
12871 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12872
12873 HM_RESTORE_PREEMPT();
12874 VMMRZCallRing3Enable(pVCpu);
12875 }
12876 }
12877
12878#ifdef VBOX_STRICT
12879 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12880 Assert(!fIOWrite);
12881 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12882 Assert(fIOWrite);
12883 else
12884 {
12885#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12886 * statuses, that the VMM device and some others may return. See
12887 * IOM_SUCCESS() for guidance. */
12888 AssertMsg( RT_FAILURE(rcStrict)
12889 || rcStrict == VINF_SUCCESS
12890 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12891 || rcStrict == VINF_EM_DBG_BREAKPOINT
12892 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12893 || rcStrict == VINF_EM_RAW_TO_R3
12894 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12895#endif
12896 }
12897#endif
12898
12899 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12900 return rcStrict;
12901}
12902
12903
12904/**
12905 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12906 * VM-exit.
12907 */
12908HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12909{
12910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12911
12912 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12913 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12914 AssertRCReturn(rc, rc);
12915 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12916 {
12917 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12918 AssertRCReturn(rc, rc);
12919 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12920 {
12921 uint32_t uErrCode;
12922 RTGCUINTPTR GCPtrFaultAddress;
12923 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12924 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12925 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12926 if (fErrorCodeValid)
12927 {
12928 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12929 AssertRCReturn(rc, rc);
12930 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12931 }
12932 else
12933 uErrCode = 0;
12934
12935 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12936 && uVector == X86_XCPT_PF)
12937 GCPtrFaultAddress = pMixedCtx->cr2;
12938 else
12939 GCPtrFaultAddress = 0;
12940
12941 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12942 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12943
12944 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12946 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12947 }
12948 }
12949
12950 /* Fall back to the interpreter to emulate the task-switch. */
12951 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12952 return VERR_EM_INTERPRETER;
12953}
12954
12955
12956/**
12957 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12958 */
12959HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12960{
12961 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12962 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12963 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12964 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12965 AssertRCReturn(rc, rc);
12966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12967 return VINF_EM_DBG_STEPPED;
12968}
12969
12970
12971/**
12972 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12973 */
12974HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12975{
12976 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12977
12978 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12979
12980 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12981 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12982 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12983 {
12984 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12985 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12986 {
12987 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12988 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12989 }
12990 }
12991 else
12992 {
12993 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12994 rcStrict1 = VINF_SUCCESS;
12995 return rcStrict1;
12996 }
12997
12998#if 0
12999 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13000 * just sync the whole thing. */
13001 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13002#else
13003 /* Aggressive state sync. for now. */
13004 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13005 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13006 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13007#endif
13008 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13009 AssertRCReturn(rc, rc);
13010
13011 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13012 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13013 VBOXSTRICTRC rcStrict2;
13014 switch (uAccessType)
13015 {
13016 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13017 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13018 {
13019 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13020 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13021 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13022
13023 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13024 GCPhys &= PAGE_BASE_GC_MASK;
13025 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13026 PVM pVM = pVCpu->CTX_SUFF(pVM);
13027 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13028 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13029
13030 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13031 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13032 CPUMCTX2CORE(pMixedCtx), GCPhys);
13033 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13034 if ( rcStrict2 == VINF_SUCCESS
13035 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13036 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13037 {
13038 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13039 | HM_CHANGED_GUEST_RSP
13040 | HM_CHANGED_GUEST_RFLAGS
13041 | HM_CHANGED_GUEST_APIC_STATE);
13042 rcStrict2 = VINF_SUCCESS;
13043 }
13044 break;
13045 }
13046
13047 default:
13048 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13049 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13050 break;
13051 }
13052
13053 if (rcStrict2 != VINF_SUCCESS)
13054 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13055 return rcStrict2;
13056}
13057
13058
13059/**
13060 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13061 * VM-exit.
13062 */
13063HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13064{
13065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13066
13067 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13068 if (pVmxTransient->fWasGuestDebugStateActive)
13069 {
13070 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13071 HMVMX_RETURN_UNEXPECTED_EXIT();
13072 }
13073
13074 if ( !pVCpu->hm.s.fSingleInstruction
13075 && !pVmxTransient->fWasHyperDebugStateActive)
13076 {
13077 Assert(!DBGFIsStepping(pVCpu));
13078 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13079
13080 /* Don't intercept MOV DRx any more. */
13081 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13082 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13083 AssertRCReturn(rc, rc);
13084
13085 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13086 VMMRZCallRing3Disable(pVCpu);
13087 HM_DISABLE_PREEMPT();
13088
13089 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13090 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13091 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13092
13093 HM_RESTORE_PREEMPT();
13094 VMMRZCallRing3Enable(pVCpu);
13095
13096#ifdef VBOX_WITH_STATISTICS
13097 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13098 AssertRCReturn(rc, rc);
13099 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13100 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13101 else
13102 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13103#endif
13104 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13105 return VINF_SUCCESS;
13106 }
13107
13108 /*
13109 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13110 * Update the segment registers and DR7 from the CPU.
13111 */
13112 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13113 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13114 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13115 AssertRCReturn(rc, rc);
13116 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13117
13118 PVM pVM = pVCpu->CTX_SUFF(pVM);
13119 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13120 {
13121 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13122 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13123 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13124 if (RT_SUCCESS(rc))
13125 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13126 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13127 }
13128 else
13129 {
13130 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13131 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13132 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13134 }
13135
13136 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13137 if (RT_SUCCESS(rc))
13138 {
13139 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13140 AssertRCReturn(rc2, rc2);
13141 return VINF_SUCCESS;
13142 }
13143 return rc;
13144}
13145
13146
13147/**
13148 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13149 * Conditional VM-exit.
13150 */
13151HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13152{
13153 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13154 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13155
13156 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13157 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13158 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13159 {
13160 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13161 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13162 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13163 {
13164 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13165 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13166 }
13167 }
13168 else
13169 {
13170 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13171 rcStrict1 = VINF_SUCCESS;
13172 return rcStrict1;
13173 }
13174
13175 RTGCPHYS GCPhys = 0;
13176 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13177
13178#if 0
13179 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13180#else
13181 /* Aggressive state sync. for now. */
13182 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13183 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13184 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13185#endif
13186 AssertRCReturn(rc, rc);
13187
13188 /*
13189 * If we succeed, resume guest execution.
13190 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13191 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13192 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13193 * weird case. See @bugref{6043}.
13194 */
13195 PVM pVM = pVCpu->CTX_SUFF(pVM);
13196 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13197 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13198 if ( rcStrict2 == VINF_SUCCESS
13199 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13200 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13201 {
13202 /* Successfully handled MMIO operation. */
13203 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13204 | HM_CHANGED_GUEST_RSP
13205 | HM_CHANGED_GUEST_RFLAGS
13206 | HM_CHANGED_GUEST_APIC_STATE);
13207 return VINF_SUCCESS;
13208 }
13209 return rcStrict2;
13210}
13211
13212
13213/**
13214 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13215 * VM-exit.
13216 */
13217HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13218{
13219 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13220 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13221
13222 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13223 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13224 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13225 {
13226 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13227 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13228 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13229 }
13230 else
13231 {
13232 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13233 rcStrict1 = VINF_SUCCESS;
13234 return rcStrict1;
13235 }
13236
13237 RTGCPHYS GCPhys = 0;
13238 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13239 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13240#if 0
13241 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13242#else
13243 /* Aggressive state sync. for now. */
13244 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13245 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13246 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13247#endif
13248 AssertRCReturn(rc, rc);
13249
13250 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13251 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13252
13253 RTGCUINT uErrorCode = 0;
13254 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13255 uErrorCode |= X86_TRAP_PF_ID;
13256 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13257 uErrorCode |= X86_TRAP_PF_RW;
13258 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13259 uErrorCode |= X86_TRAP_PF_P;
13260
13261 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13262
13263 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13264 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13265
13266 /* Handle the pagefault trap for the nested shadow table. */
13267 PVM pVM = pVCpu->CTX_SUFF(pVM);
13268 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13269 TRPMResetTrap(pVCpu);
13270
13271 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13272 if ( rcStrict2 == VINF_SUCCESS
13273 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13274 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13275 {
13276 /* Successfully synced our nested page tables. */
13277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13278 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13279 | HM_CHANGED_GUEST_RSP
13280 | HM_CHANGED_GUEST_RFLAGS);
13281 return VINF_SUCCESS;
13282 }
13283
13284 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13285 return rcStrict2;
13286}
13287
13288/** @} */
13289
13290/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13291/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13292/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13293
13294/** @name VM-exit exception handlers.
13295 * @{
13296 */
13297
13298/**
13299 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13300 */
13301static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13302{
13303 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13304 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13305
13306 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13307 AssertRCReturn(rc, rc);
13308
13309 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13310 {
13311 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13312 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13313
13314 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13315 * provides VM-exit instruction length. If this causes problem later,
13316 * disassemble the instruction like it's done on AMD-V. */
13317 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13318 AssertRCReturn(rc2, rc2);
13319 return rc;
13320 }
13321
13322 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13323 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13324 return rc;
13325}
13326
13327
13328/**
13329 * VM-exit exception handler for \#BP (Breakpoint exception).
13330 */
13331static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13332{
13333 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13335
13336 /** @todo Try optimize this by not saving the entire guest state unless
13337 * really needed. */
13338 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13339 AssertRCReturn(rc, rc);
13340
13341 PVM pVM = pVCpu->CTX_SUFF(pVM);
13342 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13343 if (rc == VINF_EM_RAW_GUEST_TRAP)
13344 {
13345 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13346 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13347 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13348 AssertRCReturn(rc, rc);
13349
13350 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13351 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13352 }
13353
13354 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13355 return rc;
13356}
13357
13358
13359/**
13360 * VM-exit exception handler for \#AC (alignment check exception).
13361 */
13362static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13363{
13364 RT_NOREF_PV(pMixedCtx);
13365 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13366
13367 /*
13368 * Re-inject it. We'll detect any nesting before getting here.
13369 */
13370 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13371 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13372 AssertRCReturn(rc, rc);
13373 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13374
13375 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13376 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13377 return VINF_SUCCESS;
13378}
13379
13380
13381/**
13382 * VM-exit exception handler for \#DB (Debug exception).
13383 */
13384static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13385{
13386 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13387 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13388 Log6(("XcptDB\n"));
13389
13390 /*
13391 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13392 * for processing.
13393 */
13394 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13395 AssertRCReturn(rc, rc);
13396
13397 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13398 uint64_t uDR6 = X86_DR6_INIT_VAL;
13399 uDR6 |= ( pVmxTransient->uExitQualification
13400 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13401
13402 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13403 if (rc == VINF_EM_RAW_GUEST_TRAP)
13404 {
13405 /*
13406 * The exception was for the guest. Update DR6, DR7.GD and
13407 * IA32_DEBUGCTL.LBR before forwarding it.
13408 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13409 */
13410 VMMRZCallRing3Disable(pVCpu);
13411 HM_DISABLE_PREEMPT();
13412
13413 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13414 pMixedCtx->dr[6] |= uDR6;
13415 if (CPUMIsGuestDebugStateActive(pVCpu))
13416 ASMSetDR6(pMixedCtx->dr[6]);
13417
13418 HM_RESTORE_PREEMPT();
13419 VMMRZCallRing3Enable(pVCpu);
13420
13421 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13422 AssertRCReturn(rc, rc);
13423
13424 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13425 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13426
13427 /* Paranoia. */
13428 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13429 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13430
13431 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13432 AssertRCReturn(rc, rc);
13433
13434 /*
13435 * Raise #DB in the guest.
13436 *
13437 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13438 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP (INT1) and not the
13439 * regular #DB. Thus it -may- trigger different handling in the CPU (like skipped DPL checks), see @bugref{6398}.
13440 *
13441 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of Intel 386,
13442 * see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13443 */
13444 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13445 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13446 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13447 AssertRCReturn(rc, rc);
13448 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13449 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13450 return VINF_SUCCESS;
13451 }
13452
13453 /*
13454 * Not a guest trap, must be a hypervisor related debug event then.
13455 * Update DR6 in case someone is interested in it.
13456 */
13457 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13458 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13459 CPUMSetHyperDR6(pVCpu, uDR6);
13460
13461 return rc;
13462}
13463
13464/**
13465 * VM-exit exception handler for \#GP (General-protection exception).
13466 *
13467 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13468 */
13469static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13470{
13471 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13472 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13473
13474 int rc;
13475 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13476 { /* likely */ }
13477 else
13478 {
13479#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13480 Assert(pVCpu->hm.s.fUsingDebugLoop);
13481#endif
13482 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13483 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13484 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13485 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13486 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13487 AssertRCReturn(rc, rc);
13488 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13489 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13490 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13491 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13492 return rc;
13493 }
13494
13495 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13496 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13497
13498 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13499 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13500 AssertRCReturn(rc, rc);
13501
13502 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13503 uint32_t cbOp = 0;
13504 PVM pVM = pVCpu->CTX_SUFF(pVM);
13505 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13506 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13507 if (RT_SUCCESS(rc))
13508 {
13509 rc = VINF_SUCCESS;
13510 Assert(cbOp == pDis->cbInstr);
13511 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13512 switch (pDis->pCurInstr->uOpcode)
13513 {
13514 case OP_CLI:
13515 {
13516 pMixedCtx->eflags.Bits.u1IF = 0;
13517 pMixedCtx->eflags.Bits.u1RF = 0;
13518 pMixedCtx->rip += pDis->cbInstr;
13519 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13520 if ( !fDbgStepping
13521 && pMixedCtx->eflags.Bits.u1TF)
13522 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13524 break;
13525 }
13526
13527 case OP_STI:
13528 {
13529 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13530 pMixedCtx->eflags.Bits.u1IF = 1;
13531 pMixedCtx->eflags.Bits.u1RF = 0;
13532 pMixedCtx->rip += pDis->cbInstr;
13533 if (!fOldIF)
13534 {
13535 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13536 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13537 }
13538 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13539 if ( !fDbgStepping
13540 && pMixedCtx->eflags.Bits.u1TF)
13541 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13542 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13543 break;
13544 }
13545
13546 case OP_HLT:
13547 {
13548 rc = VINF_EM_HALT;
13549 pMixedCtx->rip += pDis->cbInstr;
13550 pMixedCtx->eflags.Bits.u1RF = 0;
13551 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13552 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13553 break;
13554 }
13555
13556 case OP_POPF:
13557 {
13558 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13559 uint32_t cbParm;
13560 uint32_t uMask;
13561 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13562 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13563 {
13564 cbParm = 4;
13565 uMask = 0xffffffff;
13566 }
13567 else
13568 {
13569 cbParm = 2;
13570 uMask = 0xffff;
13571 }
13572
13573 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13574 RTGCPTR GCPtrStack = 0;
13575 X86EFLAGS Eflags;
13576 Eflags.u32 = 0;
13577 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13578 &GCPtrStack);
13579 if (RT_SUCCESS(rc))
13580 {
13581 Assert(sizeof(Eflags.u32) >= cbParm);
13582 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13583 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13584 }
13585 if (RT_FAILURE(rc))
13586 {
13587 rc = VERR_EM_INTERPRETER;
13588 break;
13589 }
13590 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13591 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13592 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13593 pMixedCtx->esp += cbParm;
13594 pMixedCtx->esp &= uMask;
13595 pMixedCtx->rip += pDis->cbInstr;
13596 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13597 | HM_CHANGED_GUEST_RSP
13598 | HM_CHANGED_GUEST_RFLAGS);
13599 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13600 POPF restores EFLAGS.TF. */
13601 if ( !fDbgStepping
13602 && fGstStepping)
13603 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13604 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13605 break;
13606 }
13607
13608 case OP_PUSHF:
13609 {
13610 uint32_t cbParm;
13611 uint32_t uMask;
13612 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13613 {
13614 cbParm = 4;
13615 uMask = 0xffffffff;
13616 }
13617 else
13618 {
13619 cbParm = 2;
13620 uMask = 0xffff;
13621 }
13622
13623 /* Get the stack pointer & push the contents of eflags onto the stack. */
13624 RTGCPTR GCPtrStack = 0;
13625 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13626 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13627 if (RT_FAILURE(rc))
13628 {
13629 rc = VERR_EM_INTERPRETER;
13630 break;
13631 }
13632 X86EFLAGS Eflags = pMixedCtx->eflags;
13633 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13634 Eflags.Bits.u1RF = 0;
13635 Eflags.Bits.u1VM = 0;
13636
13637 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13638 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13639 {
13640 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13641 rc = VERR_EM_INTERPRETER;
13642 break;
13643 }
13644 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13645 pMixedCtx->esp -= cbParm;
13646 pMixedCtx->esp &= uMask;
13647 pMixedCtx->rip += pDis->cbInstr;
13648 pMixedCtx->eflags.Bits.u1RF = 0;
13649 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13650 | HM_CHANGED_GUEST_RSP
13651 | HM_CHANGED_GUEST_RFLAGS);
13652 if ( !fDbgStepping
13653 && pMixedCtx->eflags.Bits.u1TF)
13654 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13656 break;
13657 }
13658
13659 case OP_IRET:
13660 {
13661 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13662 * instruction reference. */
13663 RTGCPTR GCPtrStack = 0;
13664 uint32_t uMask = 0xffff;
13665 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13666 uint16_t aIretFrame[3];
13667 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13668 {
13669 rc = VERR_EM_INTERPRETER;
13670 break;
13671 }
13672 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13673 &GCPtrStack);
13674 if (RT_SUCCESS(rc))
13675 {
13676 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13677 PGMACCESSORIGIN_HM));
13678 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13679 }
13680 if (RT_FAILURE(rc))
13681 {
13682 rc = VERR_EM_INTERPRETER;
13683 break;
13684 }
13685 pMixedCtx->eip = 0;
13686 pMixedCtx->ip = aIretFrame[0];
13687 pMixedCtx->cs.Sel = aIretFrame[1];
13688 pMixedCtx->cs.ValidSel = aIretFrame[1];
13689 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13690 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13691 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13692 pMixedCtx->sp += sizeof(aIretFrame);
13693 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13694 | HM_CHANGED_GUEST_SEGMENT_REGS
13695 | HM_CHANGED_GUEST_RSP
13696 | HM_CHANGED_GUEST_RFLAGS);
13697 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13698 if ( !fDbgStepping
13699 && fGstStepping)
13700 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13701 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13702 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13703 break;
13704 }
13705
13706 case OP_INT:
13707 {
13708 uint16_t uVector = pDis->Param1.uValue & 0xff;
13709 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13710 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13711 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13712 break;
13713 }
13714
13715 case OP_INTO:
13716 {
13717 if (pMixedCtx->eflags.Bits.u1OF)
13718 {
13719 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13720 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13721 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13722 }
13723 else
13724 {
13725 pMixedCtx->eflags.Bits.u1RF = 0;
13726 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13727 }
13728 break;
13729 }
13730
13731 default:
13732 {
13733 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13734 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13735 EMCODETYPE_SUPERVISOR);
13736 rc = VBOXSTRICTRC_VAL(rc2);
13737 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13738 /** @todo We have to set pending-debug exceptions here when the guest is
13739 * single-stepping depending on the instruction that was interpreted. */
13740 Log4(("#GP rc=%Rrc\n", rc));
13741 break;
13742 }
13743 }
13744 }
13745 else
13746 rc = VERR_EM_INTERPRETER;
13747
13748 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13749 ("#GP Unexpected rc=%Rrc\n", rc));
13750 return rc;
13751}
13752
13753
13754/**
13755 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13756 * the exception reported in the VMX transient structure back into the VM.
13757 *
13758 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13759 * up-to-date.
13760 */
13761static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13762{
13763 RT_NOREF_PV(pMixedCtx);
13764 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13765#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13766 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13767 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13768 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13769#endif
13770
13771 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13772 hmR0VmxCheckExitDueToEventDelivery(). */
13773 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13774 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13775 AssertRCReturn(rc, rc);
13776 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13777
13778#ifdef DEBUG_ramshankar
13779 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13780 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13781 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13782#endif
13783
13784 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13785 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13786 return VINF_SUCCESS;
13787}
13788
13789
13790/**
13791 * VM-exit exception handler for \#PF (Page-fault exception).
13792 */
13793static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13794{
13795 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13796 PVM pVM = pVCpu->CTX_SUFF(pVM);
13797 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13798 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13799 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13800 AssertRCReturn(rc, rc);
13801
13802 if (!pVM->hm.s.fNestedPaging)
13803 { /* likely */ }
13804 else
13805 {
13806#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13807 Assert(pVCpu->hm.s.fUsingDebugLoop);
13808#endif
13809 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13810 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13811 {
13812 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13813 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13814 }
13815 else
13816 {
13817 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13818 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13819 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13820 }
13821 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13822 return rc;
13823 }
13824
13825 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13826 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13827 if (pVmxTransient->fVectoringPF)
13828 {
13829 Assert(pVCpu->hm.s.Event.fPending);
13830 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13831 }
13832
13833 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13834 AssertRCReturn(rc, rc);
13835
13836 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13837 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13838
13839 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13840 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13841 (RTGCPTR)pVmxTransient->uExitQualification);
13842
13843 Log4(("#PF: rc=%Rrc\n", rc));
13844 if (rc == VINF_SUCCESS)
13845 {
13846#if 0
13847 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13848 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13849 * memory? We don't update the whole state here... */
13850 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13851 | HM_CHANGED_GUEST_RSP
13852 | HM_CHANGED_GUEST_RFLAGS
13853 | HM_CHANGED_GUEST_APIC_STATE);
13854#else
13855 /*
13856 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13857 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13858 */
13859 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13860 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13861#endif
13862 TRPMResetTrap(pVCpu);
13863 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13864 return rc;
13865 }
13866
13867 if (rc == VINF_EM_RAW_GUEST_TRAP)
13868 {
13869 if (!pVmxTransient->fVectoringDoublePF)
13870 {
13871 /* It's a guest page fault and needs to be reflected to the guest. */
13872 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13873 TRPMResetTrap(pVCpu);
13874 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13875 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13876 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13877 }
13878 else
13879 {
13880 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13881 TRPMResetTrap(pVCpu);
13882 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13883 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13884 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13885 }
13886
13887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13888 return VINF_SUCCESS;
13889 }
13890
13891 TRPMResetTrap(pVCpu);
13892 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13893 return rc;
13894}
13895
13896/** @} */
13897
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette