VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72531

Last change on this file since 72531 was 72531, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Comment.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 596.7 KB
Line 
1/* $Id: HMVMXR0.cpp 72531 2018-06-12 16:46:20Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#include <iprt/x86.h>
24#include <iprt/asm-amd64-x86.h>
25#include <iprt/thread.h>
26
27#include <VBox/vmm/pdmapi.h>
28#include <VBox/vmm/dbgf.h>
29#include <VBox/vmm/iem.h>
30#include <VBox/vmm/iom.h>
31#include <VBox/vmm/selm.h>
32#include <VBox/vmm/tm.h>
33#include <VBox/vmm/gim.h>
34#include <VBox/vmm/apic.h>
35#ifdef VBOX_WITH_REM
36# include <VBox/vmm/rem.h>
37#endif
38#include "HMInternal.h"
39#include <VBox/vmm/vm.h>
40#include "HMVMXR0.h"
41#include "dtrace/VBoxVMM.h"
42
43#define HMVMX_USE_IEM_EVENT_REFLECTION
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name Updated-guest-state flags.
69 * @{ */
70#define HMVMX_UPDATED_GUEST_RIP RT_BIT(0)
71#define HMVMX_UPDATED_GUEST_RSP RT_BIT(1)
72#define HMVMX_UPDATED_GUEST_RFLAGS RT_BIT(2)
73#define HMVMX_UPDATED_GUEST_CR0 RT_BIT(3)
74#define HMVMX_UPDATED_GUEST_CR3 RT_BIT(4)
75#define HMVMX_UPDATED_GUEST_CR4 RT_BIT(5)
76#define HMVMX_UPDATED_GUEST_GDTR RT_BIT(6)
77#define HMVMX_UPDATED_GUEST_IDTR RT_BIT(7)
78#define HMVMX_UPDATED_GUEST_LDTR RT_BIT(8)
79#define HMVMX_UPDATED_GUEST_TR RT_BIT(9)
80#define HMVMX_UPDATED_GUEST_SEGMENT_REGS RT_BIT(10)
81#define HMVMX_UPDATED_GUEST_DR7 RT_BIT(11)
82#define HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR RT_BIT(12)
83#define HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR RT_BIT(13)
84#define HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR RT_BIT(14)
85#define HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS RT_BIT(15)
86#define HMVMX_UPDATED_GUEST_LAZY_MSRS RT_BIT(16)
87#define HMVMX_UPDATED_GUEST_ACTIVITY_STATE RT_BIT(17)
88#define HMVMX_UPDATED_GUEST_INTR_STATE RT_BIT(18)
89#define HMVMX_UPDATED_GUEST_APIC_STATE RT_BIT(19)
90#define HMVMX_UPDATED_GUEST_ALL ( HMVMX_UPDATED_GUEST_RIP \
91 | HMVMX_UPDATED_GUEST_RSP \
92 | HMVMX_UPDATED_GUEST_RFLAGS \
93 | HMVMX_UPDATED_GUEST_CR0 \
94 | HMVMX_UPDATED_GUEST_CR3 \
95 | HMVMX_UPDATED_GUEST_CR4 \
96 | HMVMX_UPDATED_GUEST_GDTR \
97 | HMVMX_UPDATED_GUEST_IDTR \
98 | HMVMX_UPDATED_GUEST_LDTR \
99 | HMVMX_UPDATED_GUEST_TR \
100 | HMVMX_UPDATED_GUEST_SEGMENT_REGS \
101 | HMVMX_UPDATED_GUEST_DR7 \
102 | HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR \
103 | HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR \
104 | HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR \
105 | HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS \
106 | HMVMX_UPDATED_GUEST_LAZY_MSRS \
107 | HMVMX_UPDATED_GUEST_ACTIVITY_STATE \
108 | HMVMX_UPDATED_GUEST_INTR_STATE \
109 | HMVMX_UPDATED_GUEST_APIC_STATE)
110/** @} */
111
112/** @name
113 * Flags to skip redundant reads of some common VMCS fields that are not part of
114 * the guest-CPU state but are in the transient structure.
115 */
116#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO RT_BIT(0)
117#define HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE RT_BIT(1)
118#define HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION RT_BIT(2)
119#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN RT_BIT(3)
120#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO RT_BIT(4)
121#define HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE RT_BIT(5)
122#define HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO RT_BIT(6)
123/** @} */
124
125/** @name
126 * States of the VMCS.
127 *
128 * This does not reflect all possible VMCS states but currently only those
129 * needed for maintaining the VMCS consistently even when thread-context hooks
130 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
131 */
132#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
133#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
134#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
135/** @} */
136
137/**
138 * Exception bitmap mask for real-mode guests (real-on-v86).
139 *
140 * We need to intercept all exceptions manually except:
141 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
142 * due to bugs in Intel CPUs.
143 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
144 * support.
145 */
146#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
147 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
148 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
149 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
150 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
151 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
152 | RT_BIT(X86_XCPT_XF))
153
154/**
155 * Exception bitmap mask for all contributory exceptions.
156 *
157 * Page fault is deliberately excluded here as it's conditional as to whether
158 * it's contributory or benign. Page faults are handled separately.
159 */
160#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
161 | RT_BIT(X86_XCPT_DE))
162
163/** Maximum VM-instruction error number. */
164#define HMVMX_INSTR_ERROR_MAX 28
165
166/** Profiling macro. */
167#ifdef HM_PROFILE_EXIT_DISPATCH
168# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
169# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
170#else
171# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
172# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
173#endif
174
175/** Assert that preemption is disabled or covered by thread-context hooks. */
176#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
177 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
178
179/** Assert that we haven't migrated CPUs when thread-context hooks are not
180 * used. */
181#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
182 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
183 ("Illegal migration! Entered on CPU %u Current %u\n", \
184 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
185
186/** Helper macro for VM-exit handlers called unexpectedly. */
187#define HMVMX_RETURN_UNEXPECTED_EXIT() \
188 do { \
189 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
190 return VERR_VMX_UNEXPECTED_EXIT; \
191 } while (0)
192
193
194/*********************************************************************************************************************************
195* Structures and Typedefs *
196*********************************************************************************************************************************/
197/**
198 * VMX transient state.
199 *
200 * A state structure for holding miscellaneous information across
201 * VMX non-root operation and restored after the transition.
202 */
203typedef struct VMXTRANSIENT
204{
205 /** The host's rflags/eflags. */
206 RTCCUINTREG fEFlags;
207#if HC_ARCH_BITS == 32
208 uint32_t u32Alignment0;
209#endif
210 /** The guest's TPR value used for TPR shadowing. */
211 uint8_t u8GuestTpr;
212 /** Alignment. */
213 uint8_t abAlignment0[7];
214
215 /** The basic VM-exit reason. */
216 uint16_t uExitReason;
217 /** Alignment. */
218 uint16_t u16Alignment0;
219 /** The VM-exit interruption error code. */
220 uint32_t uExitIntErrorCode;
221 /** The VM-exit exit code qualification. */
222 uint64_t uExitQualification;
223
224 /** The VM-exit interruption-information field. */
225 uint32_t uExitIntInfo;
226 /** The VM-exit instruction-length field. */
227 uint32_t cbInstr;
228 /** The VM-exit instruction-information field. */
229 union
230 {
231 /** Plain unsigned int representation. */
232 uint32_t u;
233 /** INS and OUTS information. */
234 struct
235 {
236 uint32_t u7Reserved0 : 7;
237 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
238 uint32_t u3AddrSize : 3;
239 uint32_t u5Reserved1 : 5;
240 /** The segment register (X86_SREG_XXX). */
241 uint32_t iSegReg : 3;
242 uint32_t uReserved2 : 14;
243 } StrIo;
244 /** INVEPT, INVVPID, INVPCID information. */
245 struct
246 {
247 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
248 uint32_t u2Scaling : 2;
249 uint32_t u5Reserved0 : 5;
250 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
251 uint32_t u3AddrSize : 3;
252 uint32_t u1Reserved0 : 1;
253 uint32_t u4Reserved0 : 4;
254 /** The segment register (X86_SREG_XXX). */
255 uint32_t iSegReg : 3;
256 /** The index register (X86_GREG_XXX). */
257 uint32_t iIdxReg : 4;
258 /** Set if index register is invalid. */
259 uint32_t fIdxRegValid : 1;
260 /** The base register (X86_GREG_XXX). */
261 uint32_t iBaseReg : 4;
262 /** Set if base register is invalid. */
263 uint32_t fBaseRegValid : 1;
264 /** Register 2 (X86_GREG_XXX). */
265 uint32_t iReg2 : 4;
266 } Inv;
267 } ExitInstrInfo;
268 /** Whether the VM-entry failed or not. */
269 bool fVMEntryFailed;
270 /** Alignment. */
271 uint8_t abAlignment1[3];
272
273 /** The VM-entry interruption-information field. */
274 uint32_t uEntryIntInfo;
275 /** The VM-entry exception error code field. */
276 uint32_t uEntryXcptErrorCode;
277 /** The VM-entry instruction length field. */
278 uint32_t cbEntryInstr;
279
280 /** IDT-vectoring information field. */
281 uint32_t uIdtVectoringInfo;
282 /** IDT-vectoring error code. */
283 uint32_t uIdtVectoringErrorCode;
284
285 /** Mask of currently read VMCS fields; HMVMX_UPDATED_TRANSIENT_*. */
286 uint32_t fVmcsFieldsRead;
287
288 /** Whether the guest debug state was active at the time of VM-exit. */
289 bool fWasGuestDebugStateActive;
290 /** Whether the hyper debug state was active at the time of VM-exit. */
291 bool fWasHyperDebugStateActive;
292 /** Whether TSC-offsetting should be setup before VM-entry. */
293 bool fUpdateTscOffsettingAndPreemptTimer;
294 /** Whether the VM-exit was caused by a page-fault during delivery of a
295 * contributory exception or a page-fault. */
296 bool fVectoringDoublePF;
297 /** Whether the VM-exit was caused by a page-fault during delivery of an
298 * external interrupt or NMI. */
299 bool fVectoringPF;
300} VMXTRANSIENT;
301AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
302AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
303AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
304AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
305AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
306/** Pointer to VMX transient state. */
307typedef VMXTRANSIENT *PVMXTRANSIENT;
308
309
310/**
311 * MSR-bitmap read permissions.
312 */
313typedef enum VMXMSREXITREAD
314{
315 /** Reading this MSR causes a VM-exit. */
316 VMXMSREXIT_INTERCEPT_READ = 0xb,
317 /** Reading this MSR does not cause a VM-exit. */
318 VMXMSREXIT_PASSTHRU_READ
319} VMXMSREXITREAD;
320/** Pointer to MSR-bitmap read permissions. */
321typedef VMXMSREXITREAD* PVMXMSREXITREAD;
322
323/**
324 * MSR-bitmap write permissions.
325 */
326typedef enum VMXMSREXITWRITE
327{
328 /** Writing to this MSR causes a VM-exit. */
329 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
330 /** Writing to this MSR does not cause a VM-exit. */
331 VMXMSREXIT_PASSTHRU_WRITE
332} VMXMSREXITWRITE;
333/** Pointer to MSR-bitmap write permissions. */
334typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
335
336
337/**
338 * VMX VM-exit handler.
339 *
340 * @returns Strict VBox status code (i.e. informational status codes too).
341 * @param pVCpu The cross context virtual CPU structure.
342 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
343 * out-of-sync. Make sure to update the required
344 * fields before using them.
345 * @param pVmxTransient Pointer to the VMX-transient structure.
346 */
347#ifndef HMVMX_USE_FUNCTION_TABLE
348typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
349#else
350typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
351/** Pointer to VM-exit handler. */
352typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
353#endif
354
355/**
356 * VMX VM-exit handler, non-strict status code.
357 *
358 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
359 *
360 * @returns VBox status code, no informational status code returned.
361 * @param pVCpu The cross context virtual CPU structure.
362 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
363 * out-of-sync. Make sure to update the required
364 * fields before using them.
365 * @param pVmxTransient Pointer to the VMX-transient structure.
366 *
367 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
368 * use of that status code will be replaced with VINF_EM_SOMETHING
369 * later when switching over to IEM.
370 */
371#ifndef HMVMX_USE_FUNCTION_TABLE
372typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
373#else
374typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
375#endif
376
377
378/*********************************************************************************************************************************
379* Internal Functions *
380*********************************************************************************************************************************/
381static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
382static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
383static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
384static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
385 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress,
386 bool fStepping, uint32_t *puIntState);
387#if HC_ARCH_BITS == 32
388static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu);
389#endif
390#ifndef HMVMX_USE_FUNCTION_TABLE
391DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
392# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
393# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
394#else
395# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
396# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
397#endif
398
399
400/** @name VM-exit handlers.
401 * @{
402 */
403static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
404static FNVMXEXITHANDLER hmR0VmxExitExtInt;
405static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
410static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
412static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
413static FNVMXEXITHANDLER hmR0VmxExitCpuid;
414static FNVMXEXITHANDLER hmR0VmxExitGetsec;
415static FNVMXEXITHANDLER hmR0VmxExitHlt;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
417static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
418static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
419static FNVMXEXITHANDLER hmR0VmxExitVmcall;
420static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
423static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
424static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
425static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
426static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
427static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
431static FNVMXEXITHANDLER hmR0VmxExitMwait;
432static FNVMXEXITHANDLER hmR0VmxExitMtf;
433static FNVMXEXITHANDLER hmR0VmxExitMonitor;
434static FNVMXEXITHANDLER hmR0VmxExitPause;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
437static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
438static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
439static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
440static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
441static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
442static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
443static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
444static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
445static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
446static FNVMXEXITHANDLER hmR0VmxExitRdrand;
447static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
448/** @} */
449
450static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
451static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
452static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
453static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
454static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
455static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
456static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
457static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx);
458
459
460/*********************************************************************************************************************************
461* Global Variables *
462*********************************************************************************************************************************/
463#ifdef HMVMX_USE_FUNCTION_TABLE
464
465/**
466 * VMX_EXIT dispatch table.
467 */
468static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
469{
470 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
471 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
472 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
473 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
474 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
475 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
476 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
477 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
478 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
479 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
480 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
481 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
482 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
483 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
484 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
485 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
486 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
487 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
488 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
489 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
490 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
491 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
492 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
493 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
494 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
495 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
496 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
497 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
498 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
499 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
500 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
501 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
502 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
503 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
504 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
505 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
506 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
507 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
508 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
510 /* 40 UNDEFINED */ hmR0VmxExitPause,
511 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
512 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
513 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
514 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
515 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
516 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
517 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
518 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
519 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
520 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
521 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
522 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
523 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
524 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
525 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
526 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
527 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
528 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
529 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
530 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
531 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
532 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
533 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
534 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
535};
536#endif /* HMVMX_USE_FUNCTION_TABLE */
537
538#ifdef VBOX_STRICT
539static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
540{
541 /* 0 */ "(Not Used)",
542 /* 1 */ "VMCALL executed in VMX root operation.",
543 /* 2 */ "VMCLEAR with invalid physical address.",
544 /* 3 */ "VMCLEAR with VMXON pointer.",
545 /* 4 */ "VMLAUNCH with non-clear VMCS.",
546 /* 5 */ "VMRESUME with non-launched VMCS.",
547 /* 6 */ "VMRESUME after VMXOFF",
548 /* 7 */ "VM-entry with invalid control fields.",
549 /* 8 */ "VM-entry with invalid host state fields.",
550 /* 9 */ "VMPTRLD with invalid physical address.",
551 /* 10 */ "VMPTRLD with VMXON pointer.",
552 /* 11 */ "VMPTRLD with incorrect revision identifier.",
553 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
554 /* 13 */ "VMWRITE to read-only VMCS component.",
555 /* 14 */ "(Not Used)",
556 /* 15 */ "VMXON executed in VMX root operation.",
557 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
558 /* 17 */ "VM-entry with non-launched executing VMCS.",
559 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
560 /* 19 */ "VMCALL with non-clear VMCS.",
561 /* 20 */ "VMCALL with invalid VM-exit control fields.",
562 /* 21 */ "(Not Used)",
563 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
564 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
565 /* 24 */ "VMCALL with invalid SMM-monitor features.",
566 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
567 /* 26 */ "VM-entry with events blocked by MOV SS.",
568 /* 27 */ "(Not Used)",
569 /* 28 */ "Invalid operand to INVEPT/INVVPID."
570};
571#endif /* VBOX_STRICT */
572
573
574
575/**
576 * Updates the VM's last error record.
577 *
578 * If there was a VMX instruction error, reads the error data from the VMCS and
579 * updates VCPU's last error record as well.
580 *
581 * @param pVM The cross context VM structure.
582 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
583 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
584 * VERR_VMX_INVALID_VMCS_FIELD.
585 * @param rc The error code.
586 */
587static void hmR0VmxUpdateErrorRecord(PVM pVM, PVMCPU pVCpu, int rc)
588{
589 AssertPtr(pVM);
590 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
591 || rc == VERR_VMX_UNABLE_TO_START_VM)
592 {
593 AssertPtrReturnVoid(pVCpu);
594 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
595 }
596 pVM->hm.s.lLastError = rc;
597}
598
599
600/**
601 * Reads the VM-entry interruption-information field from the VMCS into the VMX
602 * transient structure.
603 *
604 * @returns VBox status code.
605 * @param pVmxTransient Pointer to the VMX transient structure.
606 *
607 * @remarks No-long-jump zone!!!
608 */
609DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
610{
611 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
612 AssertRCReturn(rc, rc);
613 return VINF_SUCCESS;
614}
615
616
617#ifdef VBOX_STRICT
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636#ifdef VBOX_STRICT
637/**
638 * Reads the VM-entry exception error code field from the VMCS into
639 * the VMX transient structure.
640 *
641 * @returns VBox status code.
642 * @param pVmxTransient Pointer to the VMX transient structure.
643 *
644 * @remarks No-long-jump zone!!!
645 */
646DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
647{
648 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
649 AssertRCReturn(rc, rc);
650 return VINF_SUCCESS;
651}
652#endif /* VBOX_STRICT */
653
654
655/**
656 * Reads the VM-exit interruption-information field from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit interruption error code from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_ERROR_CODE;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction length field from the VMCS into the VMX
695 * transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_LEN;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the VM-exit instruction-information field from the VMCS into
714 * the VMX transient structure.
715 *
716 * @returns VBox status code.
717 * @param pVmxTransient Pointer to the VMX transient structure.
718 */
719DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
720{
721 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO))
722 {
723 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
724 AssertRCReturn(rc, rc);
725 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_INSTR_INFO;
726 }
727 return VINF_SUCCESS;
728}
729
730
731/**
732 * Reads the exit code qualification from the VMCS into the VMX transient
733 * structure.
734 *
735 * @returns VBox status code.
736 * @param pVCpu The cross context virtual CPU structure of the
737 * calling EMT. (Required for the VMCS cache case.)
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 */
740DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
741{
742 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION))
743 {
744 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
745 AssertRCReturn(rc, rc);
746 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_EXIT_QUALIFICATION;
747 }
748 return VINF_SUCCESS;
749}
750
751
752/**
753 * Reads the IDT-vectoring information field from the VMCS into the VMX
754 * transient structure.
755 *
756 * @returns VBox status code.
757 * @param pVmxTransient Pointer to the VMX transient structure.
758 *
759 * @remarks No-long-jump zone!!!
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_INFO, &pVmxTransient->uIdtVectoringInfo);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_INFO;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Reads the IDT-vectoring error code from the VMCS into the VMX
775 * transient structure.
776 *
777 * @returns VBox status code.
778 * @param pVmxTransient Pointer to the VMX transient structure.
779 */
780DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
781{
782 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE))
783 {
784 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
785 AssertRCReturn(rc, rc);
786 pVmxTransient->fVmcsFieldsRead |= HMVMX_UPDATED_TRANSIENT_IDT_VECTORING_ERROR_CODE;
787 }
788 return VINF_SUCCESS;
789}
790
791
792/**
793 * Enters VMX root mode operation on the current CPU.
794 *
795 * @returns VBox status code.
796 * @param pVM The cross context VM structure. Can be
797 * NULL, after a resume.
798 * @param HCPhysCpuPage Physical address of the VMXON region.
799 * @param pvCpuPage Pointer to the VMXON region.
800 */
801static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
802{
803 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
804 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
805 Assert(pvCpuPage);
806 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
807
808 if (pVM)
809 {
810 /* Write the VMCS revision dword to the VMXON region. */
811 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
812 }
813
814 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
815 RTCCUINTREG fEFlags = ASMIntDisableFlags();
816
817 /* Enable the VMX bit in CR4 if necessary. */
818 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
819
820 /* Enter VMX root mode. */
821 int rc = VMXEnable(HCPhysCpuPage);
822 if (RT_FAILURE(rc))
823 {
824 if (!(uOldCr4 & X86_CR4_VMXE))
825 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
826
827 if (pVM)
828 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
829 }
830
831 /* Restore interrupts. */
832 ASMSetFlags(fEFlags);
833 return rc;
834}
835
836
837/**
838 * Exits VMX root mode operation on the current CPU.
839 *
840 * @returns VBox status code.
841 */
842static int hmR0VmxLeaveRootMode(void)
843{
844 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
845
846 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
847 RTCCUINTREG fEFlags = ASMIntDisableFlags();
848
849 /* If we're for some reason not in VMX root mode, then don't leave it. */
850 RTCCUINTREG uHostCR4 = ASMGetCR4();
851
852 int rc;
853 if (uHostCR4 & X86_CR4_VMXE)
854 {
855 /* Exit VMX root mode and clear the VMX bit in CR4. */
856 VMXDisable();
857 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
858 rc = VINF_SUCCESS;
859 }
860 else
861 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
862
863 /* Restore interrupts. */
864 ASMSetFlags(fEFlags);
865 return rc;
866}
867
868
869/**
870 * Allocates and maps one physically contiguous page. The allocated page is
871 * zero'd out. (Used by various VT-x structures).
872 *
873 * @returns IPRT status code.
874 * @param pMemObj Pointer to the ring-0 memory object.
875 * @param ppVirt Where to store the virtual address of the
876 * allocation.
877 * @param pHCPhys Where to store the physical address of the
878 * allocation.
879 */
880DECLINLINE(int) hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
881{
882 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
883 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
884 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
885
886 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
887 if (RT_FAILURE(rc))
888 return rc;
889 *ppVirt = RTR0MemObjAddress(*pMemObj);
890 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
891 ASMMemZero32(*ppVirt, PAGE_SIZE);
892 return VINF_SUCCESS;
893}
894
895
896/**
897 * Frees and unmaps an allocated physical page.
898 *
899 * @param pMemObj Pointer to the ring-0 memory object.
900 * @param ppVirt Where to re-initialize the virtual address of
901 * allocation as 0.
902 * @param pHCPhys Where to re-initialize the physical address of the
903 * allocation as 0.
904 */
905DECLINLINE(void) hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
906{
907 AssertPtr(pMemObj);
908 AssertPtr(ppVirt);
909 AssertPtr(pHCPhys);
910 if (*pMemObj != NIL_RTR0MEMOBJ)
911 {
912 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
913 AssertRC(rc);
914 *pMemObj = NIL_RTR0MEMOBJ;
915 *ppVirt = 0;
916 *pHCPhys = 0;
917 }
918}
919
920
921/**
922 * Worker function to free VT-x related structures.
923 *
924 * @returns IPRT status code.
925 * @param pVM The cross context VM structure.
926 */
927static void hmR0VmxStructsFree(PVM pVM)
928{
929 for (VMCPUID i = 0; i < pVM->cCpus; i++)
930 {
931 PVMCPU pVCpu = &pVM->aCpus[i];
932 AssertPtr(pVCpu);
933
934 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
935 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
936
937 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
938 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
939
940 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
941 }
942
943 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
944#ifdef VBOX_WITH_CRASHDUMP_MAGIC
945 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
946#endif
947}
948
949
950/**
951 * Worker function to allocate VT-x related VM structures.
952 *
953 * @returns IPRT status code.
954 * @param pVM The cross context VM structure.
955 */
956static int hmR0VmxStructsAlloc(PVM pVM)
957{
958 /*
959 * Initialize members up-front so we can cleanup properly on allocation failure.
960 */
961#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
962 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
963 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
964 pVM->hm.s.vmx.HCPhys##a_Name = 0;
965
966#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
967 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
968 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
969 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
970
971#ifdef VBOX_WITH_CRASHDUMP_MAGIC
972 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
973#endif
974 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
975
976 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
977 for (VMCPUID i = 0; i < pVM->cCpus; i++)
978 {
979 PVMCPU pVCpu = &pVM->aCpus[i];
980 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
981 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
982 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
983 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
984 }
985#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
986#undef VMXLOCAL_INIT_VM_MEMOBJ
987
988 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
989 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
990 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
991 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
992
993 /*
994 * Allocate all the VT-x structures.
995 */
996 int rc = VINF_SUCCESS;
997#ifdef VBOX_WITH_CRASHDUMP_MAGIC
998 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
999 if (RT_FAILURE(rc))
1000 goto cleanup;
1001 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
1002 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
1003#endif
1004
1005 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
1006 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
1007 {
1008 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
1009 &pVM->hm.s.vmx.HCPhysApicAccess);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012 }
1013
1014 /*
1015 * Initialize per-VCPU VT-x structures.
1016 */
1017 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1018 {
1019 PVMCPU pVCpu = &pVM->aCpus[i];
1020 AssertPtr(pVCpu);
1021
1022 /* Allocate the VM control structure (VMCS). */
1023 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1024 if (RT_FAILURE(rc))
1025 goto cleanup;
1026
1027 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1028 if ( PDMHasApic(pVM)
1029 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1030 {
1031 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1032 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1033 if (RT_FAILURE(rc))
1034 goto cleanup;
1035 }
1036
1037 /*
1038 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1039 * transparent accesses of specific MSRs.
1040 *
1041 * If the condition for enabling MSR bitmaps changes here, don't forget to
1042 * update HMAreMsrBitmapsAvailable().
1043 */
1044 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1045 {
1046 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1047 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1051 }
1052
1053 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1054 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1055 if (RT_FAILURE(rc))
1056 goto cleanup;
1057
1058 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1059 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1060 if (RT_FAILURE(rc))
1061 goto cleanup;
1062 }
1063
1064 return VINF_SUCCESS;
1065
1066cleanup:
1067 hmR0VmxStructsFree(pVM);
1068 return rc;
1069}
1070
1071
1072/**
1073 * Does global VT-x initialization (called during module initialization).
1074 *
1075 * @returns VBox status code.
1076 */
1077VMMR0DECL(int) VMXR0GlobalInit(void)
1078{
1079#ifdef HMVMX_USE_FUNCTION_TABLE
1080 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1081# ifdef VBOX_STRICT
1082 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1083 Assert(g_apfnVMExitHandlers[i]);
1084# endif
1085#endif
1086 return VINF_SUCCESS;
1087}
1088
1089
1090/**
1091 * Does global VT-x termination (called during module termination).
1092 */
1093VMMR0DECL(void) VMXR0GlobalTerm()
1094{
1095 /* Nothing to do currently. */
1096}
1097
1098
1099/**
1100 * Sets up and activates VT-x on the current CPU.
1101 *
1102 * @returns VBox status code.
1103 * @param pCpu Pointer to the global CPU info struct.
1104 * @param pVM The cross context VM structure. Can be
1105 * NULL after a host resume operation.
1106 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1107 * fEnabledByHost is @c true).
1108 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1109 * @a fEnabledByHost is @c true).
1110 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1111 * enable VT-x on the host.
1112 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1113 */
1114VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1115 void *pvMsrs)
1116{
1117 Assert(pCpu);
1118 Assert(pvMsrs);
1119 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1120
1121 /* Enable VT-x if it's not already enabled by the host. */
1122 if (!fEnabledByHost)
1123 {
1124 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1125 if (RT_FAILURE(rc))
1126 return rc;
1127 }
1128
1129 /*
1130 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been using EPTPs) so
1131 * we don't retain any stale guest-physical mappings which won't get invalidated when flushing by VPID.
1132 */
1133 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1134 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1135 {
1136 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1137 pCpu->fFlushAsidBeforeUse = false;
1138 }
1139 else
1140 pCpu->fFlushAsidBeforeUse = true;
1141
1142 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1143 ++pCpu->cTlbFlushes;
1144
1145 return VINF_SUCCESS;
1146}
1147
1148
1149/**
1150 * Deactivates VT-x on the current CPU.
1151 *
1152 * @returns VBox status code.
1153 * @param pCpu Pointer to the global CPU info struct.
1154 * @param pvCpuPage Pointer to the VMXON region.
1155 * @param HCPhysCpuPage Physical address of the VMXON region.
1156 *
1157 * @remarks This function should never be called when SUPR0EnableVTx() or
1158 * similar was used to enable VT-x on the host.
1159 */
1160VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1161{
1162 NOREF(pCpu);
1163 NOREF(pvCpuPage);
1164 NOREF(HCPhysCpuPage);
1165
1166 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1167 return hmR0VmxLeaveRootMode();
1168}
1169
1170
1171/**
1172 * Sets the permission bits for the specified MSR in the MSR bitmap.
1173 *
1174 * @param pVCpu The cross context virtual CPU structure.
1175 * @param uMsr The MSR value.
1176 * @param enmRead Whether reading this MSR causes a VM-exit.
1177 * @param enmWrite Whether writing this MSR causes a VM-exit.
1178 */
1179static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1180{
1181 int32_t iBit;
1182 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1183
1184 /*
1185 * Layout:
1186 * 0x000 - 0x3ff - Low MSR read bits
1187 * 0x400 - 0x7ff - High MSR read bits
1188 * 0x800 - 0xbff - Low MSR write bits
1189 * 0xc00 - 0xfff - High MSR write bits
1190 */
1191 if (uMsr <= 0x00001FFF)
1192 iBit = uMsr;
1193 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1194 {
1195 iBit = uMsr - UINT32_C(0xC0000000);
1196 pbMsrBitmap += 0x400;
1197 }
1198 else
1199 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1200
1201 Assert(iBit <= 0x1fff);
1202 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1203 ASMBitSet(pbMsrBitmap, iBit);
1204 else
1205 ASMBitClear(pbMsrBitmap, iBit);
1206
1207 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1208 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1209 else
1210 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1211}
1212
1213
1214#ifdef VBOX_STRICT
1215/**
1216 * Gets the permission bits for the specified MSR in the MSR bitmap.
1217 *
1218 * @returns VBox status code.
1219 * @retval VINF_SUCCESS if the specified MSR is found.
1220 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1221 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1222 *
1223 * @param pVCpu The cross context virtual CPU structure.
1224 * @param uMsr The MSR.
1225 * @param penmRead Where to store the read permissions.
1226 * @param penmWrite Where to store the write permissions.
1227 */
1228static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1229{
1230 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1231 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1232 int32_t iBit;
1233 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1234
1235 /* See hmR0VmxSetMsrPermission() for the layout. */
1236 if (uMsr <= 0x00001FFF)
1237 iBit = uMsr;
1238 else if ( uMsr >= 0xC0000000
1239 && uMsr <= 0xC0001FFF)
1240 {
1241 iBit = (uMsr - 0xC0000000);
1242 pbMsrBitmap += 0x400;
1243 }
1244 else
1245 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1246
1247 Assert(iBit <= 0x1fff);
1248 if (ASMBitTest(pbMsrBitmap, iBit))
1249 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1250 else
1251 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1252
1253 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1254 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1255 else
1256 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1257 return VINF_SUCCESS;
1258}
1259#endif /* VBOX_STRICT */
1260
1261
1262/**
1263 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1264 * area.
1265 *
1266 * @returns VBox status code.
1267 * @param pVCpu The cross context virtual CPU structure.
1268 * @param cMsrs The number of MSRs.
1269 */
1270DECLINLINE(int) hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1271{
1272 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1273 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1274 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1275 {
1276 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1277 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1278 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1279 }
1280
1281 /* Update number of guest MSRs to load/store across the world-switch. */
1282 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1283 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1284
1285 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1286 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1287 AssertRCReturn(rc, rc);
1288
1289 /* Update the VCPU's copy of the MSR count. */
1290 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1291
1292 return VINF_SUCCESS;
1293}
1294
1295
1296/**
1297 * Adds a new (or updates the value of an existing) guest/host MSR
1298 * pair to be swapped during the world-switch as part of the
1299 * auto-load/store MSR area in the VMCS.
1300 *
1301 * @returns VBox status code.
1302 * @param pVCpu The cross context virtual CPU structure.
1303 * @param uMsr The MSR.
1304 * @param uGuestMsrValue Value of the guest MSR.
1305 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1306 * necessary.
1307 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1308 * its value was updated. Optional, can be NULL.
1309 */
1310static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1311 bool *pfAddedAndUpdated)
1312{
1313 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1314 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1315 uint32_t i;
1316 for (i = 0; i < cMsrs; i++)
1317 {
1318 if (pGuestMsr->u32Msr == uMsr)
1319 break;
1320 pGuestMsr++;
1321 }
1322
1323 bool fAdded = false;
1324 if (i == cMsrs)
1325 {
1326 ++cMsrs;
1327 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1328 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1329
1330 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1331 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1332 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1333
1334 fAdded = true;
1335 }
1336
1337 /* Update the MSR values in the auto-load/store MSR area. */
1338 pGuestMsr->u32Msr = uMsr;
1339 pGuestMsr->u64Value = uGuestMsrValue;
1340
1341 /* Create/update the MSR slot in the host MSR area. */
1342 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1343 pHostMsr += i;
1344 pHostMsr->u32Msr = uMsr;
1345
1346 /*
1347 * Update the host MSR only when requested by the caller AND when we're
1348 * adding it to the auto-load/store area. Otherwise, it would have been
1349 * updated by hmR0VmxSaveHostMsrs(). We do this for performance reasons.
1350 */
1351 bool fUpdatedMsrValue = false;
1352 if ( fAdded
1353 && fUpdateHostMsr)
1354 {
1355 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1356 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1357 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1358 fUpdatedMsrValue = true;
1359 }
1360
1361 if (pfAddedAndUpdated)
1362 *pfAddedAndUpdated = fUpdatedMsrValue;
1363 return VINF_SUCCESS;
1364}
1365
1366
1367/**
1368 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1369 * auto-load/store MSR area in the VMCS.
1370 *
1371 * @returns VBox status code.
1372 * @param pVCpu The cross context virtual CPU structure.
1373 * @param uMsr The MSR.
1374 */
1375static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1376{
1377 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1379 for (uint32_t i = 0; i < cMsrs; i++)
1380 {
1381 /* Find the MSR. */
1382 if (pGuestMsr->u32Msr == uMsr)
1383 {
1384 /* If it's the last MSR, simply reduce the count. */
1385 if (i == cMsrs - 1)
1386 {
1387 --cMsrs;
1388 break;
1389 }
1390
1391 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1392 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1393 pLastGuestMsr += cMsrs - 1;
1394 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1395 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1396
1397 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1398 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1399 pLastHostMsr += cMsrs - 1;
1400 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1401 pHostMsr->u64Value = pLastHostMsr->u64Value;
1402 --cMsrs;
1403 break;
1404 }
1405 pGuestMsr++;
1406 }
1407
1408 /* Update the VMCS if the count changed (meaning the MSR was found). */
1409 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1410 {
1411 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1412 AssertRCReturn(rc, rc);
1413
1414 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1415 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1416 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1417
1418 Log4(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1419 return VINF_SUCCESS;
1420 }
1421
1422 return VERR_NOT_FOUND;
1423}
1424
1425
1426/**
1427 * Checks if the specified guest MSR is part of the auto-load/store area in
1428 * the VMCS.
1429 *
1430 * @returns true if found, false otherwise.
1431 * @param pVCpu The cross context virtual CPU structure.
1432 * @param uMsr The MSR to find.
1433 */
1434static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1435{
1436 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1437 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1438
1439 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1440 {
1441 if (pGuestMsr->u32Msr == uMsr)
1442 return true;
1443 }
1444 return false;
1445}
1446
1447
1448/**
1449 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1450 *
1451 * @param pVCpu The cross context virtual CPU structure.
1452 *
1453 * @remarks No-long-jump zone!!!
1454 */
1455static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1456{
1457 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1458 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1459 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1460 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1461
1462 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1463 {
1464 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1465
1466 /*
1467 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1468 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1469 */
1470 if (pHostMsr->u32Msr == MSR_K6_EFER)
1471 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1472 else
1473 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1474 }
1475
1476 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1477}
1478
1479
1480/**
1481 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1482 * perform lazy restoration of the host MSRs while leaving VT-x.
1483 *
1484 * @param pVCpu The cross context virtual CPU structure.
1485 *
1486 * @remarks No-long-jump zone!!!
1487 */
1488static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1489{
1490 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1491
1492 /*
1493 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1494 */
1495 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1496 {
1497 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1498#if HC_ARCH_BITS == 64
1499 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1500 {
1501 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1502 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1503 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1504 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1505 }
1506#endif
1507 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1508 }
1509}
1510
1511
1512/**
1513 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1514 * lazily while leaving VT-x.
1515 *
1516 * @returns true if it does, false otherwise.
1517 * @param pVCpu The cross context virtual CPU structure.
1518 * @param uMsr The MSR to check.
1519 */
1520static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1521{
1522 NOREF(pVCpu);
1523#if HC_ARCH_BITS == 64
1524 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1525 {
1526 switch (uMsr)
1527 {
1528 case MSR_K8_LSTAR:
1529 case MSR_K6_STAR:
1530 case MSR_K8_SF_MASK:
1531 case MSR_K8_KERNEL_GS_BASE:
1532 return true;
1533 }
1534 }
1535#else
1536 RT_NOREF(pVCpu, uMsr);
1537#endif
1538 return false;
1539}
1540
1541
1542/**
1543 * Saves a set of guest MSRs back into the guest-CPU context.
1544 *
1545 * @param pVCpu The cross context virtual CPU structure.
1546 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1547 * out-of-sync. Make sure to update the required fields
1548 * before using them.
1549 *
1550 * @remarks No-long-jump zone!!!
1551 */
1552static void hmR0VmxLazySaveGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1553{
1554 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1555 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1556
1557 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1558 {
1559 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1560#if HC_ARCH_BITS == 64
1561 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1562 {
1563 pMixedCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
1564 pMixedCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
1565 pMixedCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
1566 pMixedCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1567 }
1568#else
1569 NOREF(pMixedCtx);
1570#endif
1571 }
1572}
1573
1574
1575/**
1576 * Loads a set of guests MSRs to allow read/passthru to the guest.
1577 *
1578 * The name of this function is slightly confusing. This function does NOT
1579 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1580 * common prefix for functions dealing with "lazy restoration" of the shared
1581 * MSRs.
1582 *
1583 * @param pVCpu The cross context virtual CPU structure.
1584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1585 * out-of-sync. Make sure to update the required fields
1586 * before using them.
1587 *
1588 * @remarks No-long-jump zone!!!
1589 */
1590static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1596#if HC_ARCH_BITS == 64
1597 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1598 {
1599 /*
1600 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1601 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1602 * we can skip a few MSR writes.
1603 *
1604 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1605 * guest MSR values in the guest-CPU context might be different to what's currently
1606 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1607 * CPU, see @bugref{8728}.
1608 */
1609 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1610 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1611 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1612 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1613 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1614 {
1615#ifdef VBOX_STRICT
1616 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1617 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1618 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1619 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1620#endif
1621 }
1622 else
1623 {
1624 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1625 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1626 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1627 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1628 }
1629 }
1630#else
1631 RT_NOREF(pMixedCtx);
1632#endif
1633 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1634}
1635
1636
1637/**
1638 * Performs lazy restoration of the set of host MSRs if they were previously
1639 * loaded with guest MSR values.
1640 *
1641 * @param pVCpu The cross context virtual CPU structure.
1642 *
1643 * @remarks No-long-jump zone!!!
1644 * @remarks The guest MSRs should have been saved back into the guest-CPU
1645 * context by hmR0VmxSaveGuestLazyMsrs()!!!
1646 */
1647static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1648{
1649 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1650 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1651
1652 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1653 {
1654 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1655#if HC_ARCH_BITS == 64
1656 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1657 {
1658 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1659 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1660 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1661 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1662 }
1663#endif
1664 }
1665 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1666}
1667
1668
1669/**
1670 * Verifies that our cached values of the VMCS controls are all
1671 * consistent with what's actually present in the VMCS.
1672 *
1673 * @returns VBox status code.
1674 * @param pVCpu The cross context virtual CPU structure.
1675 */
1676static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1677{
1678 uint32_t u32Val;
1679 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1680 AssertRCReturn(rc, rc);
1681 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1682 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1683
1684 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1685 AssertRCReturn(rc, rc);
1686 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1687 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1688
1689 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1690 AssertRCReturn(rc, rc);
1691 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1692 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1693
1694 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1695 AssertRCReturn(rc, rc);
1696 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1697 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1698
1699 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1700 {
1701 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1702 AssertRCReturn(rc, rc);
1703 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1704 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1705 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1706 }
1707
1708 return VINF_SUCCESS;
1709}
1710
1711
1712#ifdef VBOX_STRICT
1713/**
1714 * Verifies that our cached host EFER value has not changed
1715 * since we cached it.
1716 *
1717 * @param pVCpu The cross context virtual CPU structure.
1718 */
1719static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1720{
1721 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1722
1723 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1724 {
1725 uint64_t u64Val;
1726 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1727 AssertRC(rc);
1728
1729 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1730 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1731 }
1732}
1733
1734
1735/**
1736 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1737 * VMCS are correct.
1738 *
1739 * @param pVCpu The cross context virtual CPU structure.
1740 */
1741static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1742{
1743 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1744
1745 /* Verify MSR counts in the VMCS are what we think it should be. */
1746 uint32_t cMsrs;
1747 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1748 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1749
1750 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1751 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1752
1753 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1754 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1755
1756 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1757 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1758 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1759 {
1760 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1761 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1762 pGuestMsr->u32Msr, cMsrs));
1763
1764 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1765 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1766 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1767
1768 /* Verify that the permissions are as expected in the MSR bitmap. */
1769 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1770 {
1771 VMXMSREXITREAD enmRead;
1772 VMXMSREXITWRITE enmWrite;
1773 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1774 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1775 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1776 {
1777 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1778 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1779 }
1780 else
1781 {
1782 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1783 pGuestMsr->u32Msr, cMsrs));
1784 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1785 pGuestMsr->u32Msr, cMsrs));
1786 }
1787 }
1788 }
1789}
1790#endif /* VBOX_STRICT */
1791
1792
1793/**
1794 * Flushes the TLB using EPT.
1795 *
1796 * @returns VBox status code.
1797 * @param pVCpu The cross context virtual CPU structure of the calling
1798 * EMT. Can be NULL depending on @a enmFlush.
1799 * @param enmFlush Type of flush.
1800 *
1801 * @remarks Caller is responsible for making sure this function is called only
1802 * when NestedPaging is supported and providing @a enmFlush that is
1803 * supported by the CPU.
1804 * @remarks Can be called with interrupts disabled.
1805 */
1806static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1807{
1808 uint64_t au64Descriptor[2];
1809 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1810 au64Descriptor[0] = 0;
1811 else
1812 {
1813 Assert(pVCpu);
1814 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1815 }
1816 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1817
1818 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1819 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1820 rc));
1821 if ( RT_SUCCESS(rc)
1822 && pVCpu)
1823 {
1824 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1825 }
1826}
1827
1828
1829/**
1830 * Flushes the TLB using VPID.
1831 *
1832 * @returns VBox status code.
1833 * @param pVM The cross context VM structure.
1834 * @param pVCpu The cross context virtual CPU structure of the calling
1835 * EMT. Can be NULL depending on @a enmFlush.
1836 * @param enmFlush Type of flush.
1837 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1838 * on @a enmFlush).
1839 *
1840 * @remarks Can be called with interrupts disabled.
1841 */
1842static void hmR0VmxFlushVpid(PVM pVM, PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1843{
1844 NOREF(pVM);
1845 AssertPtr(pVM);
1846 Assert(pVM->hm.s.vmx.fVpid);
1847
1848 uint64_t au64Descriptor[2];
1849 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1850 {
1851 au64Descriptor[0] = 0;
1852 au64Descriptor[1] = 0;
1853 }
1854 else
1855 {
1856 AssertPtr(pVCpu);
1857 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1858 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1859 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1860 au64Descriptor[1] = GCPtr;
1861 }
1862
1863 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]); NOREF(rc);
1864 AssertMsg(rc == VINF_SUCCESS,
1865 ("VMXR0InvVPID %#x %u %RGv failed with %d\n", enmFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1866 if ( RT_SUCCESS(rc)
1867 && pVCpu)
1868 {
1869 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1870 }
1871}
1872
1873
1874/**
1875 * Invalidates a guest page by guest virtual address. Only relevant for
1876 * EPT/VPID, otherwise there is nothing really to invalidate.
1877 *
1878 * @returns VBox status code.
1879 * @param pVM The cross context VM structure.
1880 * @param pVCpu The cross context virtual CPU structure.
1881 * @param GCVirt Guest virtual address of the page to invalidate.
1882 */
1883VMMR0DECL(int) VMXR0InvalidatePage(PVM pVM, PVMCPU pVCpu, RTGCPTR GCVirt)
1884{
1885 AssertPtr(pVM);
1886 AssertPtr(pVCpu);
1887 LogFlowFunc(("pVM=%p pVCpu=%p GCVirt=%RGv\n", pVM, pVCpu, GCVirt));
1888
1889 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1890 if (!fFlushPending)
1891 {
1892 /*
1893 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for the EPT case
1894 * See @bugref{6043} and @bugref{6177}.
1895 *
1896 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*() as this
1897 * function maybe called in a loop with individual addresses.
1898 */
1899 if (pVM->hm.s.vmx.fVpid)
1900 {
1901 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1902
1903#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1904 /*
1905 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1906 * where executing INVVPID outside 64-bit mode does not flush translations of
1907 * 64-bit linear addresses, see @bugref{6208#c72}.
1908 */
1909 if (RT_HI_U32(GCVirt))
1910 fVpidFlush = false;
1911#endif
1912
1913 if (fVpidFlush)
1914 {
1915 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1916 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1917 }
1918 else
1919 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1920 }
1921 else if (pVM->hm.s.fNestedPaging)
1922 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1923 }
1924
1925 return VINF_SUCCESS;
1926}
1927
1928
1929/**
1930 * Invalidates a guest page by physical address. Only relevant for EPT/VPID,
1931 * otherwise there is nothing really to invalidate.
1932 *
1933 * @returns VBox status code.
1934 * @param pVM The cross context VM structure.
1935 * @param pVCpu The cross context virtual CPU structure.
1936 * @param GCPhys Guest physical address of the page to invalidate.
1937 */
1938VMMR0DECL(int) VMXR0InvalidatePhysPage(PVM pVM, PVMCPU pVCpu, RTGCPHYS GCPhys)
1939{
1940 NOREF(pVM); NOREF(GCPhys);
1941 LogFlowFunc(("%RGp\n", GCPhys));
1942
1943 /*
1944 * We cannot flush a page by guest-physical address. invvpid takes only a linear address while invept only flushes
1945 * by EPT not individual addresses. We update the force flag here and flush before the next VM-entry in hmR0VmxFlushTLB*().
1946 * This function might be called in a loop. This should cause a flush-by-EPT if EPT is in use. See @bugref{6568}.
1947 */
1948 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1949 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgPhys);
1950 return VINF_SUCCESS;
1951}
1952
1953
1954/**
1955 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1956 * case where neither EPT nor VPID is supported by the CPU.
1957 *
1958 * @param pVM The cross context VM structure.
1959 * @param pVCpu The cross context virtual CPU structure.
1960 * @param pCpu Pointer to the global HM struct.
1961 *
1962 * @remarks Called with interrupts disabled.
1963 */
1964static void hmR0VmxFlushTaggedTlbNone(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1965{
1966 AssertPtr(pVCpu);
1967 AssertPtr(pCpu);
1968 NOREF(pVM);
1969
1970 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1971
1972 Assert(pCpu->idCpu != NIL_RTCPUID);
1973 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1974 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1975 pVCpu->hm.s.fForceTLBFlush = false;
1976 return;
1977}
1978
1979
1980/**
1981 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1982 *
1983 * @param pVM The cross context VM structure.
1984 * @param pVCpu The cross context virtual CPU structure.
1985 * @param pCpu Pointer to the global HM CPU struct.
1986 * @remarks All references to "ASID" in this function pertains to "VPID" in
1987 * Intel's nomenclature. The reason is, to avoid confusion in compare
1988 * statements since the host-CPU copies are named "ASID".
1989 *
1990 * @remarks Called with interrupts disabled.
1991 */
1992static void hmR0VmxFlushTaggedTlbBoth(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1993{
1994#ifdef VBOX_WITH_STATISTICS
1995 bool fTlbFlushed = false;
1996# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1997# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1998 if (!fTlbFlushed) \
1999 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
2000 } while (0)
2001#else
2002# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
2003# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
2004#endif
2005
2006 AssertPtr(pVM);
2007 AssertPtr(pCpu);
2008 AssertPtr(pVCpu);
2009 Assert(pCpu->idCpu != NIL_RTCPUID);
2010
2011 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
2012 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
2013 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
2014
2015 /*
2016 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2017 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2018 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2019 */
2020 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2021 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2022 {
2023 ++pCpu->uCurrentAsid;
2024 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2025 {
2026 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
2027 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2028 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2029 }
2030
2031 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2032 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2033 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2034
2035 /*
2036 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
2037 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
2038 */
2039 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2041 HMVMX_SET_TAGGED_TLB_FLUSHED();
2042 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
2043 }
2044
2045 /* Check for explicit TLB flushes. */
2046 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2047 {
2048 /*
2049 * Changes to the EPT paging structure by VMM requires flushing by EPT as the CPU creates
2050 * guest-physical (only EPT-tagged) mappings while traversing the EPT tables when EPT is in use.
2051 * Flushing by VPID will only flush linear (only VPID-tagged) and combined (EPT+VPID tagged) mappings
2052 * but not guest-physical mappings.
2053 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information". See @bugref{6568}.
2054 */
2055 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2056 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2057 HMVMX_SET_TAGGED_TLB_FLUSHED();
2058 }
2059
2060 pVCpu->hm.s.fForceTLBFlush = false;
2061 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2062
2063 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2064 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2065 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2066 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2067 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2068 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2069 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2070 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2071 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2072
2073 /* Update VMCS with the VPID. */
2074 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2075 AssertRC(rc);
2076
2077#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2078}
2079
2080
2081/**
2082 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2083 *
2084 * @returns VBox status code.
2085 * @param pVM The cross context VM structure.
2086 * @param pVCpu The cross context virtual CPU structure.
2087 * @param pCpu Pointer to the global HM CPU struct.
2088 *
2089 * @remarks Called with interrupts disabled.
2090 */
2091static void hmR0VmxFlushTaggedTlbEpt(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2092{
2093 AssertPtr(pVM);
2094 AssertPtr(pVCpu);
2095 AssertPtr(pCpu);
2096 Assert(pCpu->idCpu != NIL_RTCPUID);
2097 AssertMsg(pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with NestedPaging disabled."));
2098 AssertMsg(!pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID enabled."));
2099
2100 /*
2101 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2102 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2103 */
2104 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2105 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2106 {
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2109 }
2110
2111 /* Check for explicit TLB flushes. */
2112 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2113 {
2114 pVCpu->hm.s.fForceTLBFlush = true;
2115 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2116 }
2117
2118 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2119 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2120
2121 if (pVCpu->hm.s.fForceTLBFlush)
2122 {
2123 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 }
2126}
2127
2128
2129/**
2130 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2131 *
2132 * @returns VBox status code.
2133 * @param pVM The cross context VM structure.
2134 * @param pVCpu The cross context virtual CPU structure.
2135 * @param pCpu Pointer to the global HM CPU struct.
2136 *
2137 * @remarks Called with interrupts disabled.
2138 */
2139static void hmR0VmxFlushTaggedTlbVpid(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2140{
2141 AssertPtr(pVM);
2142 AssertPtr(pVCpu);
2143 AssertPtr(pCpu);
2144 Assert(pCpu->idCpu != NIL_RTCPUID);
2145 AssertMsg(pVM->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked with VPID disabled."));
2146 AssertMsg(!pVM->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging enabled"));
2147
2148 /*
2149 * Force a TLB flush for the first world switch if the current CPU differs from the one we ran on last.
2150 * If the TLB flush count changed, another VM (VCPU rather) has hit the ASID limit while flushing the TLB
2151 * or the host CPU is online after a suspend/resume, so we cannot reuse the current ASID anymore.
2152 */
2153 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2154 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2155 {
2156 pVCpu->hm.s.fForceTLBFlush = true;
2157 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2158 }
2159
2160 /* Check for explicit TLB flushes. */
2161 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2162 {
2163 /*
2164 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see hmR0VmxSetupTaggedTlb())
2165 * we would need to explicitly flush in this case (add an fExplicitFlush = true here and change the
2166 * pCpu->fFlushAsidBeforeUse check below to include fExplicitFlush's too) - an obscure corner case.
2167 */
2168 pVCpu->hm.s.fForceTLBFlush = true;
2169 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2170 }
2171
2172 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2173 if (pVCpu->hm.s.fForceTLBFlush)
2174 {
2175 ++pCpu->uCurrentAsid;
2176 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2177 {
2178 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2179 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2180 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2181 }
2182
2183 pVCpu->hm.s.fForceTLBFlush = false;
2184 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2185 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2186 if (pCpu->fFlushAsidBeforeUse)
2187 {
2188 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2189 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2190 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2191 {
2192 hmR0VmxFlushVpid(pVM, pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2193 pCpu->fFlushAsidBeforeUse = false;
2194 }
2195 else
2196 {
2197 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2198 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2199 }
2200 }
2201 }
2202
2203 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2204 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2205 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2206 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2207 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2208 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2209 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2210
2211 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2212 AssertRC(rc);
2213}
2214
2215
2216/**
2217 * Flushes the guest TLB entry based on CPU capabilities.
2218 *
2219 * @param pVCpu The cross context virtual CPU structure.
2220 * @param pCpu Pointer to the global HM CPU struct.
2221 */
2222DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2223{
2224#ifdef HMVMX_ALWAYS_FLUSH_TLB
2225 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2226#endif
2227 PVM pVM = pVCpu->CTX_SUFF(pVM);
2228 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2229 {
2230 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVM, pVCpu, pCpu); break;
2231 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVM, pVCpu, pCpu); break;
2232 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVM, pVCpu, pCpu); break;
2233 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVM, pVCpu, pCpu); break;
2234 default:
2235 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2236 break;
2237 }
2238
2239 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2240}
2241
2242
2243/**
2244 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2245 * TLB entries from the host TLB before VM-entry.
2246 *
2247 * @returns VBox status code.
2248 * @param pVM The cross context VM structure.
2249 */
2250static int hmR0VmxSetupTaggedTlb(PVM pVM)
2251{
2252 /*
2253 * Determine optimal flush type for Nested Paging.
2254 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2255 * guest execution (see hmR3InitFinalizeR0()).
2256 */
2257 if (pVM->hm.s.fNestedPaging)
2258 {
2259 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2260 {
2261 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2262 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2263 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2264 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2265 else
2266 {
2267 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2268 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2269 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2271 }
2272
2273 /* Make sure the write-back cacheable memory type for EPT is supported. */
2274 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2275 {
2276 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2277 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2278 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2279 }
2280
2281 /* EPT requires a page-walk length of 4. */
2282 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2283 {
2284 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2285 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2286 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2287 }
2288 }
2289 else
2290 {
2291 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2292 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2293 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2294 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2295 }
2296 }
2297
2298 /*
2299 * Determine optimal flush type for VPID.
2300 */
2301 if (pVM->hm.s.vmx.fVpid)
2302 {
2303 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2304 {
2305 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2306 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2307 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2308 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2309 else
2310 {
2311 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2312 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2313 LogRel(("hmR0VmxSetupTaggedTlb: Only INDIV_ADDR supported. Ignoring VPID.\n"));
2314 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2315 LogRel(("hmR0VmxSetupTaggedTlb: Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2316 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2317 pVM->hm.s.vmx.fVpid = false;
2318 }
2319 }
2320 else
2321 {
2322 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2323 Log4(("hmR0VmxSetupTaggedTlb: VPID supported without INVEPT support. Ignoring VPID.\n"));
2324 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2325 pVM->hm.s.vmx.fVpid = false;
2326 }
2327 }
2328
2329 /*
2330 * Setup the handler for flushing tagged-TLBs.
2331 */
2332 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2333 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2334 else if (pVM->hm.s.fNestedPaging)
2335 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2336 else if (pVM->hm.s.vmx.fVpid)
2337 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2338 else
2339 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2340 return VINF_SUCCESS;
2341}
2342
2343
2344/**
2345 * Sets up pin-based VM-execution controls in the VMCS.
2346 *
2347 * @returns VBox status code.
2348 * @param pVM The cross context VM structure.
2349 * @param pVCpu The cross context virtual CPU structure.
2350 */
2351static int hmR0VmxSetupPinCtls(PVM pVM, PVMCPU pVCpu)
2352{
2353 AssertPtr(pVM);
2354 AssertPtr(pVCpu);
2355
2356 uint32_t val = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2357 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2358
2359 val |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2360 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2361
2362 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2363 val |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2364
2365 /* Enable the VMX preemption timer. */
2366 if (pVM->hm.s.vmx.fUsePreemptTimer)
2367 {
2368 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2369 val |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2370 }
2371
2372#if 0
2373 /* Enable posted-interrupt processing. */
2374 if (pVM->hm.s.fPostedIntrs)
2375 {
2376 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2377 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2378 val |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2379 }
2380#endif
2381
2382 if ((val & zap) != val)
2383 {
2384 LogRel(("hmR0VmxSetupPinCtls: Invalid pin-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2385 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, val, zap));
2386 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2387 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2388 }
2389
2390 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, val);
2391 AssertRCReturn(rc, rc);
2392
2393 pVCpu->hm.s.vmx.u32PinCtls = val;
2394 return rc;
2395}
2396
2397
2398/**
2399 * Sets up processor-based VM-execution controls in the VMCS.
2400 *
2401 * @returns VBox status code.
2402 * @param pVM The cross context VM structure.
2403 * @param pVCpu The cross context virtual CPU structure.
2404 */
2405static int hmR0VmxSetupProcCtls(PVM pVM, PVMCPU pVCpu)
2406{
2407 AssertPtr(pVM);
2408 AssertPtr(pVCpu);
2409
2410 int rc = VERR_INTERNAL_ERROR_5;
2411 uint32_t val = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2412 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2413
2414 val |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2415 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2416 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2417 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2418 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2419 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2420 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2421
2422 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2423 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2424 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2425 {
2426 LogRel(("hmR0VmxSetupProcCtls: Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2427 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2428 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2429 }
2430
2431 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2432 if (!pVM->hm.s.fNestedPaging)
2433 {
2434 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2435 val |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2436 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2437 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2438 }
2439
2440 /* Use TPR shadowing if supported by the CPU. */
2441 if ( PDMHasApic(pVM)
2442 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2443 {
2444 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2445 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2446 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2447 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2448 AssertRCReturn(rc, rc);
2449
2450 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2451 /* CR8 writes cause a VM-exit based on TPR threshold. */
2452 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2453 Assert(!(val & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2454 }
2455 else
2456 {
2457 /*
2458 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2459 * Set this control only for 64-bit guests.
2460 */
2461 if (pVM->hm.s.fAllow64BitGuests)
2462 {
2463 val |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2464 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2465 }
2466 }
2467
2468 /* Use MSR-bitmaps if supported by the CPU. */
2469 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2470 {
2471 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2472
2473 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2474 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2475 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2476 AssertRCReturn(rc, rc);
2477
2478 /*
2479 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2480 * automatically using dedicated fields in the VMCS.
2481 */
2482 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2483 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2484 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2485 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2486 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2487
2488#if HC_ARCH_BITS == 64
2489 /*
2490 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2491 */
2492 if (pVM->hm.s.fAllow64BitGuests)
2493 {
2494 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2495 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2496 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2497 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2498 }
2499#endif
2500 /*
2501 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2502 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2503 */
2504 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506
2507 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2508 }
2509
2510 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2511 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2512 val |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2513
2514 if ((val & zap) != val)
2515 {
2516 LogRel(("hmR0VmxSetupProcCtls: Invalid processor-based VM-execution controls combo! cpu=%#RX64 val=%#RX64 zap=%#RX64\n",
2517 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, val, zap));
2518 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2519 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2520 }
2521
2522 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, val);
2523 AssertRCReturn(rc, rc);
2524
2525 pVCpu->hm.s.vmx.u32ProcCtls = val;
2526
2527 /*
2528 * Secondary processor-based VM-execution controls.
2529 */
2530 if (RT_LIKELY(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL))
2531 {
2532 val = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2533 zap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2534
2535 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2536 val |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT; /* WBINVD causes a VM-exit. */
2537
2538 if (pVM->hm.s.fNestedPaging)
2539 val |= VMX_VMCS_CTRL_PROC_EXEC2_EPT; /* Enable EPT. */
2540
2541 /*
2542 * Enable the INVPCID instruction if supported by the hardware and we expose
2543 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2544 */
2545 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2546 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2547 {
2548 val |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2549 }
2550
2551 if (pVM->hm.s.vmx.fVpid)
2552 val |= VMX_VMCS_CTRL_PROC_EXEC2_VPID; /* Enable VPID. */
2553
2554 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2555 val |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST; /* Enable Unrestricted Execution. */
2556
2557#if 0
2558 if (pVM->hm.s.fVirtApicRegs)
2559 {
2560 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2561 val |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT; /* Enable APIC-register virtualization. */
2562
2563 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2564 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY; /* Enable virtual-interrupt delivery. */
2565 }
2566#endif
2567
2568 /* Enable Virtual-APIC page accesses if supported by the CPU. This is essentially where the TPR shadow resides. */
2569 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2570 * done dynamically. */
2571 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2572 {
2573 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2574 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2575 val |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2576 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2577 AssertRCReturn(rc, rc);
2578 }
2579
2580 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2581 val |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP; /* Enable RDTSCP support. */
2582
2583 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2584 && pVM->hm.s.vmx.cPleGapTicks
2585 && pVM->hm.s.vmx.cPleWindowTicks)
2586 {
2587 val |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT; /* Enable pause-loop exiting. */
2588
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2591 AssertRCReturn(rc, rc);
2592 }
2593
2594 if ((val & zap) != val)
2595 {
2596 LogRel(("hmR0VmxSetupProcCtls: Invalid secondary processor-based VM-execution controls combo! "
2597 "cpu=%#RX64 val=%#RX64 zap=%#RX64\n", pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, val, zap));
2598 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2599 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2600 }
2601
2602 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, val);
2603 AssertRCReturn(rc, rc);
2604
2605 pVCpu->hm.s.vmx.u32ProcCtls2 = val;
2606 }
2607 else if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2608 {
2609 LogRel(("hmR0VmxSetupProcCtls: Unrestricted Guest set as true when secondary processor-based VM-execution controls not "
2610 "available\n"));
2611 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2612 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2613 }
2614
2615 return VINF_SUCCESS;
2616}
2617
2618
2619/**
2620 * Sets up miscellaneous (everything other than Pin & Processor-based
2621 * VM-execution) control fields in the VMCS.
2622 *
2623 * @returns VBox status code.
2624 * @param pVM The cross context VM structure.
2625 * @param pVCpu The cross context virtual CPU structure.
2626 */
2627static int hmR0VmxSetupMiscCtls(PVM pVM, PVMCPU pVCpu)
2628{
2629 NOREF(pVM);
2630 AssertPtr(pVM);
2631 AssertPtr(pVCpu);
2632
2633 int rc = VERR_GENERAL_FAILURE;
2634
2635 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2636#if 0
2637 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxLoadGuestCR3AndCR4())*/
2638 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2639 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2640
2641 /*
2642 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2643 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2644 * We thus use the exception bitmap to control it rather than use both.
2645 */
2646 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2647 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2648
2649 /** @todo Explore possibility of using IO-bitmaps. */
2650 /* All IO & IOIO instructions cause VM-exits. */
2651 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2652 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2653
2654 /* Initialize the MSR-bitmap area. */
2655 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2656 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2657 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2658 AssertRCReturn(rc, rc);
2659#endif
2660
2661 /* Setup MSR auto-load/store area. */
2662 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2663 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2664 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2665 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2666 AssertRCReturn(rc, rc);
2667
2668 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2669 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2670 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2671 AssertRCReturn(rc, rc);
2672
2673 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2674 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2675 AssertRCReturn(rc, rc);
2676
2677 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2678#if 0
2679 /* Setup debug controls */
2680 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0); /** @todo We don't support IA32_DEBUGCTL MSR. Should we? */
2681 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2682 AssertRCReturn(rc, rc);
2683#endif
2684
2685 return rc;
2686}
2687
2688
2689/**
2690 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2691 *
2692 * We shall setup those exception intercepts that don't change during the
2693 * lifetime of the VM here. The rest are done dynamically while loading the
2694 * guest state.
2695 *
2696 * @returns VBox status code.
2697 * @param pVM The cross context VM structure.
2698 * @param pVCpu The cross context virtual CPU structure.
2699 */
2700static int hmR0VmxInitXcptBitmap(PVM pVM, PVMCPU pVCpu)
2701{
2702 AssertPtr(pVM);
2703 AssertPtr(pVCpu);
2704
2705 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
2706
2707 uint32_t u32XcptBitmap = 0;
2708
2709 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2710 u32XcptBitmap |= RT_BIT_32(X86_XCPT_AC);
2711
2712 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2713 and writes, and because recursive #DBs can cause the CPU hang, we must always
2714 intercept #DB. */
2715 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2716
2717 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2718 if (!pVM->hm.s.fNestedPaging)
2719 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2720
2721 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2722 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2723 AssertRCReturn(rc, rc);
2724 return rc;
2725}
2726
2727
2728/**
2729 * Sets up the initial guest-state mask. The guest-state mask is consulted
2730 * before reading guest-state fields from the VMCS as VMREADs can be expensive
2731 * for the nested virtualization case (as it would cause a VM-exit).
2732 *
2733 * @param pVCpu The cross context virtual CPU structure.
2734 */
2735static int hmR0VmxInitUpdatedGuestStateMask(PVMCPU pVCpu)
2736{
2737 /* Initially the guest-state is up-to-date as there is nothing in the VMCS. */
2738 HMVMXCPU_GST_RESET_TO(pVCpu, HMVMX_UPDATED_GUEST_ALL);
2739 return VINF_SUCCESS;
2740}
2741
2742
2743/**
2744 * Does per-VM VT-x initialization.
2745 *
2746 * @returns VBox status code.
2747 * @param pVM The cross context VM structure.
2748 */
2749VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2750{
2751 LogFlowFunc(("pVM=%p\n", pVM));
2752
2753 int rc = hmR0VmxStructsAlloc(pVM);
2754 if (RT_FAILURE(rc))
2755 {
2756 LogRel(("VMXR0InitVM: hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2757 return rc;
2758 }
2759
2760 return VINF_SUCCESS;
2761}
2762
2763
2764/**
2765 * Does per-VM VT-x termination.
2766 *
2767 * @returns VBox status code.
2768 * @param pVM The cross context VM structure.
2769 */
2770VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2771{
2772 LogFlowFunc(("pVM=%p\n", pVM));
2773
2774#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2775 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2776 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2777#endif
2778 hmR0VmxStructsFree(pVM);
2779 return VINF_SUCCESS;
2780}
2781
2782
2783/**
2784 * Sets up the VM for execution under VT-x.
2785 * This function is only called once per-VM during initialization.
2786 *
2787 * @returns VBox status code.
2788 * @param pVM The cross context VM structure.
2789 */
2790VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2791{
2792 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2794
2795 LogFlowFunc(("pVM=%p\n", pVM));
2796
2797 /*
2798 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be allocated.
2799 * We no longer support the highly unlikely case of UnrestrictedGuest without pRealModeTSS. See hmR3InitFinalizeR0Intel().
2800 */
2801 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2802 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2803 || !pVM->hm.s.vmx.pRealModeTSS))
2804 {
2805 LogRel(("VMXR0SetupVM: Invalid real-on-v86 state.\n"));
2806 return VERR_INTERNAL_ERROR;
2807 }
2808
2809 /* Initialize these always, see hmR3InitFinalizeR0().*/
2810 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2811 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2812
2813 /* Setup the tagged-TLB flush handlers. */
2814 int rc = hmR0VmxSetupTaggedTlb(pVM);
2815 if (RT_FAILURE(rc))
2816 {
2817 LogRel(("VMXR0SetupVM: hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2818 return rc;
2819 }
2820
2821 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2822 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2823#if HC_ARCH_BITS == 64
2824 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2825 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2826 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2827 {
2828 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2829 }
2830#endif
2831
2832 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2833 RTCCUINTREG uHostCR4 = ASMGetCR4();
2834 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2835 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2836
2837 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2838 {
2839 PVMCPU pVCpu = &pVM->aCpus[i];
2840 AssertPtr(pVCpu);
2841 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2842
2843 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2844 Log4(("VMXR0SetupVM: pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2845
2846 /* Initialize the VM-exit history array with end-of-array markers (UINT16_MAX). */
2847 Assert(!pVCpu->hm.s.idxExitHistoryFree);
2848 HMCPU_EXIT_HISTORY_RESET(pVCpu);
2849
2850 /* Set revision dword at the beginning of the VMCS structure. */
2851 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2852
2853 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2854 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2855 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2856 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2857
2858 /* Load this VMCS as the current VMCS. */
2859 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2860 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2861 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2862
2863 rc = hmR0VmxSetupPinCtls(pVM, pVCpu);
2864 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2865 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2866
2867 rc = hmR0VmxSetupProcCtls(pVM, pVCpu);
2868 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2869 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2870
2871 rc = hmR0VmxSetupMiscCtls(pVM, pVCpu);
2872 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2873 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2874
2875 rc = hmR0VmxInitXcptBitmap(pVM, pVCpu);
2876 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2877 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2878
2879 rc = hmR0VmxInitUpdatedGuestStateMask(pVCpu);
2880 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitUpdatedGuestStateMask failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2881 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2882
2883#if HC_ARCH_BITS == 32
2884 rc = hmR0VmxInitVmcsReadCache(pVM, pVCpu);
2885 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2886 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2887#endif
2888
2889 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2890 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2891 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc (pVM=%p)\n", rc, pVM),
2892 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc), rc);
2893
2894 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2895
2896 hmR0VmxUpdateErrorRecord(pVM, pVCpu, rc);
2897 }
2898
2899 return VINF_SUCCESS;
2900}
2901
2902
2903/**
2904 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2905 * the VMCS.
2906 *
2907 * @returns VBox status code.
2908 * @param pVM The cross context VM structure.
2909 * @param pVCpu The cross context virtual CPU structure.
2910 */
2911DECLINLINE(int) hmR0VmxSaveHostControlRegs(PVM pVM, PVMCPU pVCpu)
2912{
2913 NOREF(pVM); NOREF(pVCpu);
2914
2915 RTCCUINTREG uReg = ASMGetCR0();
2916 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2917 AssertRCReturn(rc, rc);
2918
2919 uReg = ASMGetCR3();
2920 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2921 AssertRCReturn(rc, rc);
2922
2923 uReg = ASMGetCR4();
2924 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2925 AssertRCReturn(rc, rc);
2926 return rc;
2927}
2928
2929
2930#if HC_ARCH_BITS == 64
2931/**
2932 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2933 * requirements. See hmR0VmxSaveHostSegmentRegs().
2934 */
2935# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2936 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2937 { \
2938 bool fValidSelector = true; \
2939 if ((selValue) & X86_SEL_LDT) \
2940 { \
2941 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2942 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2943 } \
2944 if (fValidSelector) \
2945 { \
2946 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2947 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2948 } \
2949 (selValue) = 0; \
2950 }
2951#endif
2952
2953
2954/**
2955 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2956 * the host-state area in the VMCS.
2957 *
2958 * @returns VBox status code.
2959 * @param pVM The cross context VM structure.
2960 * @param pVCpu The cross context virtual CPU structure.
2961 */
2962DECLINLINE(int) hmR0VmxSaveHostSegmentRegs(PVM pVM, PVMCPU pVCpu)
2963{
2964 int rc = VERR_INTERNAL_ERROR_5;
2965
2966#if HC_ARCH_BITS == 64
2967 /*
2968 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2969 * should -not- save the messed up state without restoring the original host-state. See @bugref{7240}.
2970 *
2971 * This apparently can happen (most likely the FPU changes), deal with it rather than asserting.
2972 * Was observed booting Solaris10u10 32-bit guest.
2973 */
2974 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2975 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2976 {
2977 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2978 pVCpu->idCpu));
2979 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2980 }
2981 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2982#else
2983 RT_NOREF(pVCpu);
2984#endif
2985
2986 /*
2987 * Host DS, ES, FS and GS segment registers.
2988 */
2989#if HC_ARCH_BITS == 64
2990 RTSEL uSelDS = ASMGetDS();
2991 RTSEL uSelES = ASMGetES();
2992 RTSEL uSelFS = ASMGetFS();
2993 RTSEL uSelGS = ASMGetGS();
2994#else
2995 RTSEL uSelDS = 0;
2996 RTSEL uSelES = 0;
2997 RTSEL uSelFS = 0;
2998 RTSEL uSelGS = 0;
2999#endif
3000
3001 /*
3002 * Host CS and SS segment registers.
3003 */
3004 RTSEL uSelCS = ASMGetCS();
3005 RTSEL uSelSS = ASMGetSS();
3006
3007 /*
3008 * Host TR segment register.
3009 */
3010 RTSEL uSelTR = ASMGetTR();
3011
3012#if HC_ARCH_BITS == 64
3013 /*
3014 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to gain VM-entry and restore them
3015 * before we get preempted. See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
3016 */
3017 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
3018 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
3019 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
3020 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
3021# undef VMXLOCAL_ADJUST_HOST_SEG
3022#endif
3023
3024 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
3025 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
3026 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
3027 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
3028 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
3029 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
3030 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
3031 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
3032 Assert(uSelCS);
3033 Assert(uSelTR);
3034
3035 /* Assertion is right but we would not have updated u32ExitCtls yet. */
3036#if 0
3037 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
3038 Assert(uSelSS != 0);
3039#endif
3040
3041 /* Write these host selector fields into the host-state area in the VMCS. */
3042 rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
3043 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
3044#if HC_ARCH_BITS == 64
3045 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
3046 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
3047 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
3048 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
3049#else
3050 NOREF(uSelDS);
3051 NOREF(uSelES);
3052 NOREF(uSelFS);
3053 NOREF(uSelGS);
3054#endif
3055 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
3056 AssertRCReturn(rc, rc);
3057
3058 /*
3059 * Host GDTR and IDTR.
3060 */
3061 RTGDTR Gdtr;
3062 RTIDTR Idtr;
3063 RT_ZERO(Gdtr);
3064 RT_ZERO(Idtr);
3065 ASMGetGDTR(&Gdtr);
3066 ASMGetIDTR(&Idtr);
3067 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3068 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3069 AssertRCReturn(rc, rc);
3070
3071#if HC_ARCH_BITS == 64
3072 /*
3073 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps them to the
3074 * maximum limit (0xffff) on every VM-exit.
3075 */
3076 if (Gdtr.cbGdt != 0xffff)
3077 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3078
3079 /*
3080 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT"
3081 * and Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit as 0xfff, VT-x
3082 * bloating the limit to 0xffff shouldn't cause any different CPU behavior. However, several hosts either insists
3083 * on 0xfff being the limit (Windows Patch Guard) or uses the limit for other purposes (darwin puts the CPU ID in there
3084 * but botches sidt alignment in at least one consumer). So, we're only allowing IDTR.LIMIT to be left at 0xffff on
3085 * hosts where we are pretty sure it won't cause trouble.
3086 */
3087# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3088 if (Idtr.cbIdt < 0x0fff)
3089# else
3090 if (Idtr.cbIdt != 0xffff)
3091# endif
3092 {
3093 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3094 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3095 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3096 }
3097#endif
3098
3099 /*
3100 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI and RPL bits
3101 * is effectively what the CPU does for "scaling by 8". TI is always 0 and RPL should be too in most cases.
3102 */
3103 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3104 ("hmR0VmxSaveHostSegmentRegs: TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt),
3105 VERR_VMX_INVALID_HOST_STATE);
3106
3107 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3108#if HC_ARCH_BITS == 64
3109 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3110
3111 /*
3112 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on all VM-exits.
3113 * The type is the same for 64-bit busy TSS[1]. The limit needs manual restoration if the host has something else.
3114 * Task switching is not supported in 64-bit mode[2], but the limit still matters as IOPM is supported in 64-bit mode.
3115 * Restoring the limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3116 *
3117 * [1] See Intel spec. 3.5 "System Descriptor Types".
3118 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3119 */
3120 Assert(pDesc->System.u4Type == 11);
3121 if ( pDesc->System.u16LimitLow != 0x67
3122 || pDesc->System.u4LimitHigh)
3123 {
3124 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3125 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3126 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3127 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3128 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3129 }
3130
3131 /*
3132 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3133 */
3134 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3135 {
3136 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3137 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3138 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3139 {
3140 /* The GDT is read-only but the writable GDT is available. */
3141 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3142 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3143 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3144 AssertRCReturn(rc, rc);
3145 }
3146 }
3147#else
3148 NOREF(pVM);
3149 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3150#endif
3151 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3152 AssertRCReturn(rc, rc);
3153
3154 /*
3155 * Host FS base and GS base.
3156 */
3157#if HC_ARCH_BITS == 64
3158 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3159 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3160 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3161 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3162 AssertRCReturn(rc, rc);
3163
3164 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3165 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3166 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3167 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3168 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3169#endif
3170 return rc;
3171}
3172
3173
3174/**
3175 * Saves certain host MSRs in the VM-exit MSR-load area and some in the
3176 * host-state area of the VMCS. Theses MSRs will be automatically restored on
3177 * the host after every successful VM-exit.
3178 *
3179 * @returns VBox status code.
3180 * @param pVM The cross context VM structure.
3181 * @param pVCpu The cross context virtual CPU structure.
3182 *
3183 * @remarks No-long-jump zone!!!
3184 */
3185DECLINLINE(int) hmR0VmxSaveHostMsrs(PVM pVM, PVMCPU pVCpu)
3186{
3187 NOREF(pVM);
3188
3189 AssertPtr(pVCpu);
3190 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3191
3192 /*
3193 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3194 * rather than swapping them on every VM-entry.
3195 */
3196 hmR0VmxLazySaveHostMsrs(pVCpu);
3197
3198 /*
3199 * Host Sysenter MSRs.
3200 */
3201 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3202#if HC_ARCH_BITS == 32
3203 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3204 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3205#else
3206 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3207 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3208#endif
3209 AssertRCReturn(rc, rc);
3210
3211 /*
3212 * Host EFER MSR.
3213 * If the CPU supports the newer VMCS controls for managing EFER, use it.
3214 * Otherwise it's done as part of auto-load/store MSR area in the VMCS, see hmR0VmxLoadGuestMsrs().
3215 */
3216 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3217 {
3218 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3219 AssertRCReturn(rc, rc);
3220 }
3221
3222 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see
3223 * hmR0VmxLoadGuestExitCtls() !! */
3224
3225 return rc;
3226}
3227
3228
3229/**
3230 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3231 *
3232 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3233 * these two bits are handled by VM-entry, see hmR0VmxLoadGuestExitCtls() and
3234 * hmR0VMxLoadGuestEntryCtls().
3235 *
3236 * @returns true if we need to load guest EFER, false otherwise.
3237 * @param pVCpu The cross context virtual CPU structure.
3238 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3239 * out-of-sync. Make sure to update the required fields
3240 * before using them.
3241 *
3242 * @remarks Requires EFER, CR4.
3243 * @remarks No-long-jump zone!!!
3244 */
3245static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3246{
3247#ifdef HMVMX_ALWAYS_SWAP_EFER
3248 return true;
3249#endif
3250
3251#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3252 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3253 if (CPUMIsGuestInLongMode(pVCpu))
3254 return false;
3255#endif
3256
3257 PVM pVM = pVCpu->CTX_SUFF(pVM);
3258 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3259 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3260
3261 /*
3262 * For 64-bit guests, if EFER.SCE bit differs, we need to swap to ensure that the
3263 * guest's SYSCALL behaviour isn't screwed. See @bugref{7386}.
3264 */
3265 if ( CPUMIsGuestInLongMode(pVCpu)
3266 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3267 {
3268 return true;
3269 }
3270
3271 /*
3272 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3273 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3274 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3275 */
3276 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3277 && (pMixedCtx->cr0 & X86_CR0_PG)
3278 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3279 {
3280 /* Assert that host is PAE capable. */
3281 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3282 return true;
3283 }
3284
3285 /** @todo Check the latest Intel spec. for any other bits,
3286 * like SMEP/SMAP? */
3287 return false;
3288}
3289
3290
3291/**
3292 * Sets up VM-entry controls in the VMCS. These controls can affect things done
3293 * on VM-exit; e.g. "load debug controls", see Intel spec. 24.8.1 "VM-entry
3294 * controls".
3295 *
3296 * @returns VBox status code.
3297 * @param pVCpu The cross context virtual CPU structure.
3298 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3299 * out-of-sync. Make sure to update the required fields
3300 * before using them.
3301 *
3302 * @remarks Requires EFER.
3303 * @remarks No-long-jump zone!!!
3304 */
3305DECLINLINE(int) hmR0VmxLoadGuestEntryCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3306{
3307 int rc = VINF_SUCCESS;
3308 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS))
3309 {
3310 PVM pVM = pVCpu->CTX_SUFF(pVM);
3311 uint32_t val = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3312 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3313
3314 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3315 val |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3316
3317 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3318 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3319 {
3320 val |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3321 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n", pVCpu->idCpu));
3322 }
3323 else
3324 Assert(!(val & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3325
3326 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3327 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3328 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3329 {
3330 val |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3331 Log4(("Load[%RU32]: VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n", pVCpu->idCpu));
3332 }
3333
3334 /*
3335 * The following should -not- be set (since we're not in SMM mode):
3336 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3337 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3338 */
3339
3340 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3341 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3342
3343 if ((val & zap) != val)
3344 {
3345 LogRel(("hmR0VmxLoadGuestEntryCtls: Invalid VM-entry controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3346 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, val, zap));
3347 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3348 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3349 }
3350
3351 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, val);
3352 AssertRCReturn(rc, rc);
3353
3354 pVCpu->hm.s.vmx.u32EntryCtls = val;
3355 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_ENTRY_CTLS);
3356 }
3357 return rc;
3358}
3359
3360
3361/**
3362 * Sets up the VM-exit controls in the VMCS.
3363 *
3364 * @returns VBox status code.
3365 * @param pVCpu The cross context virtual CPU structure.
3366 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3367 * out-of-sync. Make sure to update the required fields
3368 * before using them.
3369 *
3370 * @remarks Requires EFER.
3371 */
3372DECLINLINE(int) hmR0VmxLoadGuestExitCtls(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3373{
3374 NOREF(pMixedCtx);
3375
3376 int rc = VINF_SUCCESS;
3377 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_EXIT_CTLS))
3378 {
3379 PVM pVM = pVCpu->CTX_SUFF(pVM);
3380 uint32_t val = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3381 uint32_t zap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3382
3383 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3384 val |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3385
3386 /*
3387 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3388 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in hmR0VmxSaveHostMsrs().
3389 */
3390#if HC_ARCH_BITS == 64
3391 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3392 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3393#else
3394 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3395 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3396 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3397 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3398 {
3399 /* The switcher returns to long mode, EFER is managed by the switcher. */
3400 val |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3401 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n", pVCpu->idCpu));
3402 }
3403 else
3404 Assert(!(val & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3405#endif
3406
3407 /* If the newer VMCS fields for managing EFER exists, use it. */
3408 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3409 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3410 {
3411 val |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3412 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3413 Log4(("Load[%RU32]: VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR, VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n", pVCpu->idCpu));
3414 }
3415
3416 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3417 Assert(!(val & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3418
3419 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3420 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3421 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3422
3423 if ( pVM->hm.s.vmx.fUsePreemptTimer
3424 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3425 val |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3426
3427 if ((val & zap) != val)
3428 {
3429 LogRel(("hmR0VmxSetupProcCtls: Invalid VM-exit controls combo! cpu=%RX64 val=%RX64 zap=%RX64\n",
3430 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, val, zap));
3431 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3432 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3433 }
3434
3435 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, val);
3436 AssertRCReturn(rc, rc);
3437
3438 pVCpu->hm.s.vmx.u32ExitCtls = val;
3439 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_EXIT_CTLS);
3440 }
3441 return rc;
3442}
3443
3444
3445/**
3446 * Sets the TPR threshold in the VMCS.
3447 *
3448 * @returns VBox status code.
3449 * @param pVCpu The cross context virtual CPU structure.
3450 * @param u32TprThreshold The TPR threshold (task-priority class only).
3451 */
3452DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3453{
3454 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3455 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3456 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3457}
3458
3459
3460/**
3461 * Loads the guest APIC and related state.
3462 *
3463 * @returns VBox status code.
3464 * @param pVCpu The cross context virtual CPU structure.
3465 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3466 * out-of-sync. Make sure to update the required fields
3467 * before using them.
3468 *
3469 * @remarks No-long-jump zone!!!
3470 */
3471DECLINLINE(int) hmR0VmxLoadGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3472{
3473 NOREF(pMixedCtx);
3474
3475 int rc = VINF_SUCCESS;
3476 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_APIC_STATE))
3477 {
3478 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3479 && APICIsEnabled(pVCpu))
3480 {
3481 /*
3482 * Setup TPR shadowing.
3483 */
3484 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3485 {
3486 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3487
3488 bool fPendingIntr = false;
3489 uint8_t u8Tpr = 0;
3490 uint8_t u8PendingIntr = 0;
3491 rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3492 AssertRCReturn(rc, rc);
3493
3494 /*
3495 * If there are interrupts pending but masked by the TPR, instruct VT-x to cause a TPR-below-threshold VM-exit
3496 * when the guest lowers its TPR below the priority of the pending interrupt so we can deliver the interrupt.
3497 * If there are no interrupts pending, set threshold to 0 to not cause any TPR-below-threshold VM-exits.
3498 */
3499 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3500 uint32_t u32TprThreshold = 0;
3501 if (fPendingIntr)
3502 {
3503 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3504 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3505 const uint8_t u8TprPriority = u8Tpr >> 4;
3506 if (u8PendingPriority <= u8TprPriority)
3507 u32TprThreshold = u8PendingPriority;
3508 }
3509
3510 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3511 AssertRCReturn(rc, rc);
3512 }
3513 }
3514 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
3515 }
3516
3517 return rc;
3518}
3519
3520
3521/**
3522 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3523 *
3524 * @returns Guest's interruptibility-state.
3525 * @param pVCpu The cross context virtual CPU structure.
3526 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3527 * out-of-sync. Make sure to update the required fields
3528 * before using them.
3529 *
3530 * @remarks No-long-jump zone!!!
3531 */
3532DECLINLINE(uint32_t) hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3533{
3534 /*
3535 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3536 */
3537 uint32_t uIntrState = 0;
3538 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3539 {
3540 /* If inhibition is active, RIP & RFLAGS should've been accessed (i.e. read previously from the VMCS or from ring-3). */
3541 AssertMsg(HMVMXCPU_GST_IS_SET(pVCpu, HMVMX_UPDATED_GUEST_RIP | HMVMX_UPDATED_GUEST_RFLAGS),
3542 ("%#x\n", HMVMXCPU_GST_VALUE(pVCpu)));
3543 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3544 {
3545 if (pMixedCtx->eflags.Bits.u1IF)
3546 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3547 else
3548 uIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3549 }
3550 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3551 {
3552 /*
3553 * We can clear the inhibit force flag as even if we go back to the recompiler without executing guest code in
3554 * VT-x, the flag's condition to be cleared is met and thus the cleared state is correct.
3555 */
3556 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3557 }
3558 }
3559
3560 /*
3561 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3562 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3563 * setting this would block host-NMIs and IRET will not clear the blocking.
3564 *
3565 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3566 */
3567 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3568 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3569 {
3570 uIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3571 }
3572
3573 return uIntrState;
3574}
3575
3576
3577/**
3578 * Loads the guest's interruptibility-state into the guest-state area in the
3579 * VMCS.
3580 *
3581 * @returns VBox status code.
3582 * @param pVCpu The cross context virtual CPU structure.
3583 * @param uIntrState The interruptibility-state to set.
3584 */
3585static int hmR0VmxLoadGuestIntrState(PVMCPU pVCpu, uint32_t uIntrState)
3586{
3587 NOREF(pVCpu);
3588 AssertMsg(!(uIntrState & 0xfffffff0), ("%#x\n", uIntrState)); /* Bits 31:4 MBZ. */
3589 Assert((uIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3590 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, uIntrState);
3591 AssertRC(rc);
3592 return rc;
3593}
3594
3595
3596/**
3597 * Loads the exception intercepts required for guest execution in the VMCS.
3598 *
3599 * @returns VBox status code.
3600 * @param pVCpu The cross context virtual CPU structure.
3601 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3602 * out-of-sync. Make sure to update the required fields
3603 * before using them.
3604 */
3605static int hmR0VmxLoadGuestXcptIntercepts(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3606{
3607 NOREF(pMixedCtx);
3608 int rc = VINF_SUCCESS;
3609 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
3610 {
3611 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxLoadSharedCR0(). */
3612 if (pVCpu->hm.s.fGIMTrapXcptUD)
3613 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3614#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3615 else
3616 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3617#endif
3618
3619 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3620 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3621
3622 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3623 AssertRCReturn(rc, rc);
3624
3625 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3626 Log4(("Load[%RU32]: VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu,
3627 pVCpu->hm.s.vmx.u32XcptBitmap, HMCPU_CF_VALUE(pVCpu)));
3628 }
3629 return rc;
3630}
3631
3632
3633/**
3634 * Loads the guest's RIP into the guest-state area in the VMCS.
3635 *
3636 * @returns VBox status code.
3637 * @param pVCpu The cross context virtual CPU structure.
3638 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3639 * out-of-sync. Make sure to update the required fields
3640 * before using them.
3641 *
3642 * @remarks No-long-jump zone!!!
3643 */
3644static int hmR0VmxLoadGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3645{
3646 int rc = VINF_SUCCESS;
3647 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP))
3648 {
3649 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3650 AssertRCReturn(rc, rc);
3651
3652 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RIP);
3653 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RIP=%#RX64 fContextUseFlags=%#RX32\n", pVCpu->idCpu, pMixedCtx->rip,
3654 HMCPU_CF_VALUE(pVCpu)));
3655 }
3656 return rc;
3657}
3658
3659
3660/**
3661 * Loads the guest's RSP into the guest-state area in the VMCS.
3662 *
3663 * @returns VBox status code.
3664 * @param pVCpu The cross context virtual CPU structure.
3665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3666 * out-of-sync. Make sure to update the required fields
3667 * before using them.
3668 *
3669 * @remarks No-long-jump zone!!!
3670 */
3671static int hmR0VmxLoadGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3672{
3673 int rc = VINF_SUCCESS;
3674 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP))
3675 {
3676 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3677 AssertRCReturn(rc, rc);
3678
3679 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RSP);
3680 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RSP=%#RX64\n", pVCpu->idCpu, pMixedCtx->rsp));
3681 }
3682 return rc;
3683}
3684
3685
3686/**
3687 * Loads the guest's RFLAGS into the guest-state area in the VMCS.
3688 *
3689 * @returns VBox status code.
3690 * @param pVCpu The cross context virtual CPU structure.
3691 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3692 * out-of-sync. Make sure to update the required fields
3693 * before using them.
3694 *
3695 * @remarks No-long-jump zone!!!
3696 */
3697static int hmR0VmxLoadGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3698{
3699 int rc = VINF_SUCCESS;
3700 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
3701 {
3702 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3703 Let us assert it as such and use 32-bit VMWRITE. */
3704 Assert(!(pMixedCtx->rflags.u64 >> 32));
3705 X86EFLAGS Eflags = pMixedCtx->eflags;
3706 /** @todo r=bird: There shall be no need to OR in X86_EFL_1 here, nor
3707 * shall there be any reason for clearing bits 63:22, 15, 5 and 3.
3708 * These will never be cleared/set, unless some other part of the VMM
3709 * code is buggy - in which case we're better of finding and fixing
3710 * those bugs than hiding them. */
3711 Assert(Eflags.u32 & X86_EFL_RA1_MASK);
3712 Assert(!(Eflags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3713 Eflags.u32 &= VMX_EFLAGS_RESERVED_0; /* Bits 22-31, 15, 5 & 3 MBZ. */
3714 Eflags.u32 |= VMX_EFLAGS_RESERVED_1; /* Bit 1 MB1. */
3715
3716 /*
3717 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so we can restore them on VM-exit.
3718 * Modify the real-mode guest's eflags so that VT-x can run the real-mode guest code under Virtual 8086 mode.
3719 */
3720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721 {
3722 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3723 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3724 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = Eflags.u32; /* Save the original eflags of the real-mode guest. */
3725 Eflags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3726 Eflags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3727 }
3728
3729 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, Eflags.u32);
3730 AssertRCReturn(rc, rc);
3731
3732 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_RFLAGS);
3733 Log4(("Load[%RU32]: VMX_VMCS_GUEST_RFLAGS=%#RX32\n", pVCpu->idCpu, Eflags.u32));
3734 }
3735 return rc;
3736}
3737
3738
3739/**
3740 * Loads the guest RIP, RSP and RFLAGS into the guest-state area in the VMCS.
3741 *
3742 * @returns VBox status code.
3743 * @param pVCpu The cross context virtual CPU structure.
3744 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3745 * out-of-sync. Make sure to update the required fields
3746 * before using them.
3747 *
3748 * @remarks No-long-jump zone!!!
3749 */
3750DECLINLINE(int) hmR0VmxLoadGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3751{
3752 int rc = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
3753 rc |= hmR0VmxLoadGuestRsp(pVCpu, pMixedCtx);
3754 rc |= hmR0VmxLoadGuestRflags(pVCpu, pMixedCtx);
3755 AssertRCReturn(rc, rc);
3756 return rc;
3757}
3758
3759
3760/**
3761 * Loads the guest CR0 control register into the guest-state area in the VMCS.
3762 * CR0 is partially shared with the host and we have to consider the FPU bits.
3763 *
3764 * @returns VBox status code.
3765 * @param pVCpu The cross context virtual CPU structure.
3766 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3767 * out-of-sync. Make sure to update the required fields
3768 * before using them.
3769 *
3770 * @remarks No-long-jump zone!!!
3771 */
3772static int hmR0VmxLoadSharedCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3773{
3774 Assert(CPUMIsGuestFPUStateActive(pVCpu));
3775
3776 /*
3777 * Guest CR0.
3778 * Guest FPU.
3779 */
3780 int rc = VINF_SUCCESS;
3781 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
3782 {
3783 Assert(!(pMixedCtx->cr0 >> 32));
3784 uint32_t u32GuestCR0 = pMixedCtx->cr0;
3785 PVM pVM = pVCpu->CTX_SUFF(pVM);
3786
3787 /* The guest's view (read access) of its CR0 is unblemished. */
3788 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32GuestCR0);
3789 AssertRCReturn(rc, rc);
3790 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR0));
3791
3792 /* Setup VT-x's view of the guest CR0. */
3793 /* Minimize VM-exits due to CR3 changes when we have NestedPaging. */
3794 if (pVM->hm.s.fNestedPaging)
3795 {
3796 if (CPUMIsGuestPagingEnabledEx(pMixedCtx))
3797 {
3798 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3799 pVCpu->hm.s.vmx.u32ProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3800 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3801 }
3802 else
3803 {
3804 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3805 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3806 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3807 }
3808
3809 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3810 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3811 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3812
3813 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
3814 AssertRCReturn(rc, rc);
3815 }
3816 else
3817 u32GuestCR0 |= X86_CR0_WP; /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3818
3819 /*
3820 * Guest FPU bits.
3821 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be set on the first
3822 * CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3823 */
3824 u32GuestCR0 |= X86_CR0_NE;
3825
3826 /* Catch floating point exceptions if we need to report them to the guest in a different way. */
3827 bool fInterceptMF = false;
3828 if (!(pMixedCtx->cr0 & X86_CR0_NE))
3829 fInterceptMF = true;
3830
3831 /* Finally, intercept all exceptions as we cannot directly inject them in real-mode, see hmR0VmxInjectEventVmcs(). */
3832 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3833 {
3834 Assert(PDMVmmDevHeapIsEnabled(pVM));
3835 Assert(pVM->hm.s.vmx.pRealModeTSS);
3836 pVCpu->hm.s.vmx.u32XcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3837 }
3838 else
3839 {
3840 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3841 pVCpu->hm.s.vmx.u32XcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3842 if (fInterceptMF)
3843 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_MF);
3844 }
3845 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
3846
3847 /* Additional intercepts for debugging, define these yourself explicitly. */
3848#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3849 pVCpu->hm.s.vmx.u32XcptBitmap |= 0
3850 | RT_BIT(X86_XCPT_BP)
3851 | RT_BIT(X86_XCPT_DE)
3852 | RT_BIT(X86_XCPT_NM)
3853 | RT_BIT(X86_XCPT_TS)
3854 | RT_BIT(X86_XCPT_UD)
3855 | RT_BIT(X86_XCPT_NP)
3856 | RT_BIT(X86_XCPT_SS)
3857 | RT_BIT(X86_XCPT_GP)
3858 | RT_BIT(X86_XCPT_PF)
3859 | RT_BIT(X86_XCPT_MF)
3860 ;
3861#elif defined(HMVMX_ALWAYS_TRAP_PF)
3862 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
3863#endif
3864
3865 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3866
3867 /* Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW). */
3868 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3869 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3870 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3871 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3872 else
3873 Assert((uSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3874
3875 u32GuestCR0 |= uSetCR0;
3876 u32GuestCR0 &= uZapCR0;
3877 u32GuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3878
3879 /* Write VT-x's view of the guest CR0 into the VMCS. */
3880 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCR0);
3881 AssertRCReturn(rc, rc);
3882 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR0=%#RX32 (uSetCR0=%#RX32 uZapCR0=%#RX32)\n", pVCpu->idCpu, u32GuestCR0, uSetCR0,
3883 uZapCR0));
3884
3885 /*
3886 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3887 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3888 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3889 */
3890 uint32_t u32CR0Mask = 0;
3891 u32CR0Mask = X86_CR0_PE
3892 | X86_CR0_NE
3893 | X86_CR0_WP
3894 | X86_CR0_PG
3895 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3896 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3897 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3898
3899 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3900 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3901 * and @bugref{6944}. */
3902#if 0
3903 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3904 u32CR0Mask &= ~X86_CR0_PE;
3905#endif
3906 if (pVM->hm.s.fNestedPaging)
3907 u32CR0Mask &= ~X86_CR0_WP;
3908
3909 /* Write the CR0 mask into the VMCS and update the VCPU's copy of the current CR0 mask. */
3910 pVCpu->hm.s.vmx.u32CR0Mask = u32CR0Mask;
3911 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32CR0Mask);
3912 AssertRCReturn(rc, rc);
3913 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR0_MASK=%#RX32\n", pVCpu->idCpu, u32CR0Mask));
3914
3915 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR0);
3916 }
3917 return rc;
3918}
3919
3920
3921/**
3922 * Loads the guest control registers (CR3, CR4) into the guest-state area
3923 * in the VMCS.
3924 *
3925 * @returns VBox strict status code.
3926 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3927 * without unrestricted guest access and the VMMDev is not presently
3928 * mapped (e.g. EFI32).
3929 *
3930 * @param pVCpu The cross context virtual CPU structure.
3931 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3932 * out-of-sync. Make sure to update the required fields
3933 * before using them.
3934 *
3935 * @remarks No-long-jump zone!!!
3936 */
3937static VBOXSTRICTRC hmR0VmxLoadGuestCR3AndCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3938{
3939 int rc = VINF_SUCCESS;
3940 PVM pVM = pVCpu->CTX_SUFF(pVM);
3941
3942 /*
3943 * Guest CR2.
3944 * It's always loaded in the assembler code. Nothing to do here.
3945 */
3946
3947 /*
3948 * Guest CR3.
3949 */
3950 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3))
3951 {
3952 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3953 if (pVM->hm.s.fNestedPaging)
3954 {
3955 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3956
3957 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3958 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3959 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3960 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3961
3962 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3963 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3964 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3965
3966 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3967 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3968 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3969 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3970 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3971 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3972 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3973
3974 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3975 AssertRCReturn(rc, rc);
3976 Log4(("Load[%RU32]: VMX_VMCS64_CTRL_EPTP_FULL=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.vmx.HCPhysEPTP));
3977
3978 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3979 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3980 {
3981 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3982 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3983 {
3984 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3985 AssertRCReturn(rc, rc);
3986 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3987 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3988 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3989 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3990 AssertRCReturn(rc, rc);
3991 }
3992
3993 /* The guest's view of its CR3 is unblemished with Nested Paging when the guest is using paging or we
3994 have Unrestricted Execution to handle the guest when it's not using paging. */
3995 GCPhysGuestCR3 = pMixedCtx->cr3;
3996 }
3997 else
3998 {
3999 /*
4000 * The guest is not using paging, but the CPU (VT-x) has to. While the guest thinks it accesses physical memory
4001 * directly, we use our identity-mapped page table to map guest-linear to guest-physical addresses.
4002 * EPT takes care of translating it to host-physical addresses.
4003 */
4004 RTGCPHYS GCPhys;
4005 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
4006
4007 /* We obtain it here every time as the guest could have relocated this PCI region. */
4008 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
4009 if (RT_SUCCESS(rc))
4010 { /* likely */ }
4011 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
4012 {
4013 Log4(("Load[%RU32]: VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n", pVCpu->idCpu));
4014 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
4015 }
4016 else
4017 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
4018
4019 GCPhysGuestCR3 = GCPhys;
4020 }
4021
4022 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RGp (GstN)\n", pVCpu->idCpu, GCPhysGuestCR3));
4023 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
4024 }
4025 else
4026 {
4027 /* Non-nested paging case, just use the hypervisor's CR3. */
4028 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
4029
4030 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR3=%#RHv (HstN)\n", pVCpu->idCpu, HCPhysGuestCR3));
4031 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
4032 }
4033 AssertRCReturn(rc, rc);
4034
4035 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR3);
4036 }
4037
4038 /*
4039 * Guest CR4.
4040 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
4041 */
4042 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4))
4043 {
4044 Assert(!(pMixedCtx->cr4 >> 32));
4045 uint32_t u32GuestCR4 = pMixedCtx->cr4;
4046
4047 /* The guest's view of its CR4 is unblemished. */
4048 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32GuestCR4);
4049 AssertRCReturn(rc, rc);
4050 Log4(("Load[%RU32]: VMX_VMCS_CTRL_CR4_READ_SHADOW=%#RX32\n", pVCpu->idCpu, u32GuestCR4));
4051
4052 /* Setup VT-x's view of the guest CR4. */
4053 /*
4054 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software interrupts to the 8086 program
4055 * interrupt handler. Clear the VME bit (the interrupt redirection bitmap is already all 0, see hmR3InitFinalizeR0())
4056 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
4057 */
4058 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4059 {
4060 Assert(pVM->hm.s.vmx.pRealModeTSS);
4061 Assert(PDMVmmDevHeapIsEnabled(pVM));
4062 u32GuestCR4 &= ~X86_CR4_VME;
4063 }
4064
4065 if (pVM->hm.s.fNestedPaging)
4066 {
4067 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4068 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4069 {
4070 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4071 u32GuestCR4 |= X86_CR4_PSE;
4072 /* Our identity mapping is a 32-bit page directory. */
4073 u32GuestCR4 &= ~X86_CR4_PAE;
4074 }
4075 /* else use guest CR4.*/
4076 }
4077 else
4078 {
4079 /*
4080 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4081 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4082 */
4083 switch (pVCpu->hm.s.enmShadowMode)
4084 {
4085 case PGMMODE_REAL: /* Real-mode. */
4086 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4087 case PGMMODE_32_BIT: /* 32-bit paging. */
4088 {
4089 u32GuestCR4 &= ~X86_CR4_PAE;
4090 break;
4091 }
4092
4093 case PGMMODE_PAE: /* PAE paging. */
4094 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4095 {
4096 u32GuestCR4 |= X86_CR4_PAE;
4097 break;
4098 }
4099
4100 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4101 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4102#ifdef VBOX_ENABLE_64_BITS_GUESTS
4103 break;
4104#endif
4105 default:
4106 AssertFailed();
4107 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4108 }
4109 }
4110
4111 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4112 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4113 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4114 u32GuestCR4 |= uSetCR4;
4115 u32GuestCR4 &= uZapCR4;
4116
4117 /* Write VT-x's view of the guest CR4 into the VMCS. */
4118 Log4(("Load[%RU32]: VMX_VMCS_GUEST_CR4=%#RX32 (Set=%#RX32 Zap=%#RX32)\n", pVCpu->idCpu, u32GuestCR4, uSetCR4, uZapCR4));
4119 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCR4);
4120 AssertRCReturn(rc, rc);
4121
4122 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them, that would cause a VM-exit. */
4123 uint32_t u32CR4Mask = X86_CR4_VME
4124 | X86_CR4_PAE
4125 | X86_CR4_PGE
4126 | X86_CR4_PSE
4127 | X86_CR4_VMXE;
4128 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4129 u32CR4Mask |= X86_CR4_OSXSAVE;
4130 if (pVM->cpum.ro.GuestFeatures.fPcid)
4131 u32CR4Mask |= X86_CR4_PCIDE;
4132 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4133 rc = VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4134 AssertRCReturn(rc, rc);
4135
4136 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4137 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4138
4139 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR4);
4140 }
4141 return rc;
4142}
4143
4144
4145/**
4146 * Loads the guest debug registers into the guest-state area in the VMCS.
4147 *
4148 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4149 *
4150 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4151 *
4152 * @returns VBox status code.
4153 * @param pVCpu The cross context virtual CPU structure.
4154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4155 * out-of-sync. Make sure to update the required fields
4156 * before using them.
4157 *
4158 * @remarks No-long-jump zone!!!
4159 */
4160static int hmR0VmxLoadSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4161{
4162 if (!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
4163 return VINF_SUCCESS;
4164
4165#ifdef VBOX_STRICT
4166 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4167 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4168 {
4169 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4170 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4171 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4172 }
4173#endif
4174
4175 int rc;
4176 PVM pVM = pVCpu->CTX_SUFF(pVM);
4177 bool fSteppingDB = false;
4178 bool fInterceptMovDRx = false;
4179 if (pVCpu->hm.s.fSingleInstruction)
4180 {
4181 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4182 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4183 {
4184 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4185 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4186 AssertRCReturn(rc, rc);
4187 Assert(fSteppingDB == false);
4188 }
4189 else
4190 {
4191 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4192 pVCpu->hm.s.fClearTrapFlag = true;
4193 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
4194 fSteppingDB = true;
4195 }
4196 }
4197
4198 if ( fSteppingDB
4199 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4200 {
4201 /*
4202 * Use the combined guest and host DRx values found in the hypervisor
4203 * register set because the debugger has breakpoints active or someone
4204 * is single stepping on the host side without a monitor trap flag.
4205 *
4206 * Note! DBGF expects a clean DR6 state before executing guest code.
4207 */
4208#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4209 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4210 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4211 {
4212 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4213 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4214 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4215 }
4216 else
4217#endif
4218 if (!CPUMIsHyperDebugStateActive(pVCpu))
4219 {
4220 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4221 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4222 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4223 }
4224
4225 /* Update DR7. (The other DRx values are handled by CPUM one way or the other.) */
4226 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)CPUMGetHyperDR7(pVCpu));
4227 AssertRCReturn(rc, rc);
4228
4229 pVCpu->hm.s.fUsingHyperDR7 = true;
4230 fInterceptMovDRx = true;
4231 }
4232 else
4233 {
4234 /*
4235 * If the guest has enabled debug registers, we need to load them prior to
4236 * executing guest code so they'll trigger at the right time.
4237 */
4238 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD)) /** @todo Why GD? */
4239 {
4240#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4241 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4242 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4243 {
4244 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4245 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4246 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4247 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4248 }
4249 else
4250#endif
4251 if (!CPUMIsGuestDebugStateActive(pVCpu))
4252 {
4253 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4254 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4255 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4256 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4257 }
4258 Assert(!fInterceptMovDRx);
4259 }
4260 /*
4261 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4262 * must intercept #DB in order to maintain a correct DR6 guest value, and
4263 * because we need to intercept it to prevent nested #DBs from hanging the
4264 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4265 */
4266#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4267 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4268 && !CPUMIsGuestDebugStateActive(pVCpu))
4269#else
4270 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4271#endif
4272 {
4273 fInterceptMovDRx = true;
4274 }
4275
4276 /* Update guest DR7. */
4277 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, pMixedCtx->dr[7]);
4278 AssertRCReturn(rc, rc);
4279
4280 pVCpu->hm.s.fUsingHyperDR7 = false;
4281 }
4282
4283 /*
4284 * Update the processor-based VM-execution controls regarding intercepting MOV DRx instructions.
4285 */
4286 if (fInterceptMovDRx)
4287 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4288 else
4289 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4290 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
4291 AssertRCReturn(rc, rc);
4292
4293 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_DEBUG);
4294 return VINF_SUCCESS;
4295}
4296
4297
4298#ifdef VBOX_STRICT
4299/**
4300 * Strict function to validate segment registers.
4301 *
4302 * @remarks ASSUMES CR0 is up to date.
4303 */
4304static void hmR0VmxValidateSegmentRegs(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
4305{
4306 /* Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers". */
4307 /* NOTE: The reason we check for attribute value 0 and not just the unusable bit here is because hmR0VmxWriteSegmentReg()
4308 * only updates the VMCS' copy of the value with the unusable bit and doesn't change the guest-context value. */
4309 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4310 && ( !CPUMIsGuestInRealModeEx(pCtx)
4311 && !CPUMIsGuestInV86ModeEx(pCtx)))
4312 {
4313 /* Protected mode checks */
4314 /* CS */
4315 Assert(pCtx->cs.Attr.n.u1Present);
4316 Assert(!(pCtx->cs.Attr.u & 0xf00));
4317 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4318 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4319 || !(pCtx->cs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4321 || (pCtx->cs.Attr.n.u1Granularity));
4322 /* CS cannot be loaded with NULL in protected mode. */
4323 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4324 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4325 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4326 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4327 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4328 else
4329 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4330 /* SS */
4331 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4332 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4333 if ( !(pCtx->cr0 & X86_CR0_PE)
4334 || pCtx->cs.Attr.n.u4Type == 3)
4335 {
4336 Assert(!pCtx->ss.Attr.n.u2Dpl);
4337 }
4338 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4339 {
4340 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4341 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4342 Assert(pCtx->ss.Attr.n.u1Present);
4343 Assert(!(pCtx->ss.Attr.u & 0xf00));
4344 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4345 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4346 || !(pCtx->ss.Attr.n.u1Granularity));
4347 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4348 || (pCtx->ss.Attr.n.u1Granularity));
4349 }
4350 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4351 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4352 {
4353 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4354 Assert(pCtx->ds.Attr.n.u1Present);
4355 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4356 Assert(!(pCtx->ds.Attr.u & 0xf00));
4357 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4358 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4359 || !(pCtx->ds.Attr.n.u1Granularity));
4360 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4361 || (pCtx->ds.Attr.n.u1Granularity));
4362 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4363 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4364 }
4365 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4366 {
4367 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4368 Assert(pCtx->es.Attr.n.u1Present);
4369 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4370 Assert(!(pCtx->es.Attr.u & 0xf00));
4371 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4372 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4373 || !(pCtx->es.Attr.n.u1Granularity));
4374 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4375 || (pCtx->es.Attr.n.u1Granularity));
4376 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4377 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4378 }
4379 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4380 {
4381 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4382 Assert(pCtx->fs.Attr.n.u1Present);
4383 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4384 Assert(!(pCtx->fs.Attr.u & 0xf00));
4385 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4386 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4387 || !(pCtx->fs.Attr.n.u1Granularity));
4388 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4389 || (pCtx->fs.Attr.n.u1Granularity));
4390 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4391 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4392 }
4393 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4394 {
4395 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4396 Assert(pCtx->gs.Attr.n.u1Present);
4397 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4398 Assert(!(pCtx->gs.Attr.u & 0xf00));
4399 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4400 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4401 || !(pCtx->gs.Attr.n.u1Granularity));
4402 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4403 || (pCtx->gs.Attr.n.u1Granularity));
4404 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4405 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4406 }
4407 /* 64-bit capable CPUs. */
4408# if HC_ARCH_BITS == 64
4409 Assert(!(pCtx->cs.u64Base >> 32));
4410 Assert(!pCtx->ss.Attr.u || !(pCtx->ss.u64Base >> 32));
4411 Assert(!pCtx->ds.Attr.u || !(pCtx->ds.u64Base >> 32));
4412 Assert(!pCtx->es.Attr.u || !(pCtx->es.u64Base >> 32));
4413# endif
4414 }
4415 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4416 || ( CPUMIsGuestInRealModeEx(pCtx)
4417 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4418 {
4419 /* Real and v86 mode checks. */
4420 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4421 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4422 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4423 {
4424 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4425 }
4426 else
4427 {
4428 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4429 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4430 }
4431
4432 /* CS */
4433 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4434 Assert(pCtx->cs.u32Limit == 0xffff);
4435 Assert(u32CSAttr == 0xf3);
4436 /* SS */
4437 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4438 Assert(pCtx->ss.u32Limit == 0xffff);
4439 Assert(u32SSAttr == 0xf3);
4440 /* DS */
4441 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4442 Assert(pCtx->ds.u32Limit == 0xffff);
4443 Assert(u32DSAttr == 0xf3);
4444 /* ES */
4445 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4446 Assert(pCtx->es.u32Limit == 0xffff);
4447 Assert(u32ESAttr == 0xf3);
4448 /* FS */
4449 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4450 Assert(pCtx->fs.u32Limit == 0xffff);
4451 Assert(u32FSAttr == 0xf3);
4452 /* GS */
4453 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4454 Assert(pCtx->gs.u32Limit == 0xffff);
4455 Assert(u32GSAttr == 0xf3);
4456 /* 64-bit capable CPUs. */
4457# if HC_ARCH_BITS == 64
4458 Assert(!(pCtx->cs.u64Base >> 32));
4459 Assert(!u32SSAttr || !(pCtx->ss.u64Base >> 32));
4460 Assert(!u32DSAttr || !(pCtx->ds.u64Base >> 32));
4461 Assert(!u32ESAttr || !(pCtx->es.u64Base >> 32));
4462# endif
4463 }
4464}
4465#endif /* VBOX_STRICT */
4466
4467
4468/**
4469 * Writes a guest segment register into the guest-state area in the VMCS.
4470 *
4471 * @returns VBox status code.
4472 * @param pVCpu The cross context virtual CPU structure.
4473 * @param idxSel Index of the selector in the VMCS.
4474 * @param idxLimit Index of the segment limit in the VMCS.
4475 * @param idxBase Index of the segment base in the VMCS.
4476 * @param idxAccess Index of the access rights of the segment in the VMCS.
4477 * @param pSelReg Pointer to the segment selector.
4478 *
4479 * @remarks No-long-jump zone!!!
4480 */
4481static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4482 uint32_t idxAccess, PCPUMSELREG pSelReg)
4483{
4484 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4485 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4486 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4487 AssertRCReturn(rc, rc);
4488
4489 uint32_t u32Access = pSelReg->Attr.u;
4490 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4491 {
4492 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4493 u32Access = 0xf3;
4494 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4495 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4496 }
4497 else
4498 {
4499 /*
4500 * The way to differentiate between whether this is really a null selector or was just a selector loaded with 0 in
4501 * real-mode is using the segment attributes. A selector loaded in real-mode with the value 0 is valid and usable in
4502 * protected-mode and we should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures NULL selectors
4503 * loaded in protected-mode have their attribute as 0.
4504 */
4505 if (!u32Access)
4506 u32Access = X86DESCATTR_UNUSABLE;
4507 }
4508
4509 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4510 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4511 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4512
4513 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4514 AssertRCReturn(rc, rc);
4515 return rc;
4516}
4517
4518
4519/**
4520 * Loads the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4521 * into the guest-state area in the VMCS.
4522 *
4523 * @returns VBox status code.
4524 * @param pVCpu The cross context virtual CPU structure.
4525 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4526 * out-of-sync. Make sure to update the required fields
4527 * before using them.
4528 *
4529 * @remarks ASSUMES pMixedCtx->cr0 is up to date (strict builds validation).
4530 * @remarks No-long-jump zone!!!
4531 */
4532static int hmR0VmxLoadGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4533{
4534 int rc = VERR_INTERNAL_ERROR_5;
4535 PVM pVM = pVCpu->CTX_SUFF(pVM);
4536
4537 /*
4538 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4539 */
4540 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS))
4541 {
4542 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4543 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4544 {
4545 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4546 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4547 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4548 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4549 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4550 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4551 }
4552
4553#ifdef VBOX_WITH_REM
4554 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4555 {
4556 Assert(pVM->hm.s.vmx.pRealModeTSS);
4557 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4558 if ( pVCpu->hm.s.vmx.fWasInRealMode
4559 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4560 {
4561 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4562 in real-mode (e.g. OpenBSD 4.0) */
4563 REMFlushTBs(pVM);
4564 Log4(("Load[%RU32]: Switch to protected mode detected!\n", pVCpu->idCpu));
4565 pVCpu->hm.s.vmx.fWasInRealMode = false;
4566 }
4567 }
4568#endif
4569 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4570 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4571 AssertRCReturn(rc, rc);
4572 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4573 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4574 AssertRCReturn(rc, rc);
4575 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4576 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4577 AssertRCReturn(rc, rc);
4578 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4579 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4580 AssertRCReturn(rc, rc);
4581 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4582 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4583 AssertRCReturn(rc, rc);
4584 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4585 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4586 AssertRCReturn(rc, rc);
4587
4588#ifdef VBOX_STRICT
4589 /* Validate. */
4590 hmR0VmxValidateSegmentRegs(pVM, pVCpu, pMixedCtx);
4591#endif
4592
4593 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS);
4594 Log4(("Load[%RU32]: CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pVCpu->idCpu, pMixedCtx->cs.Sel,
4595 pMixedCtx->cs.u64Base, pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4596 }
4597
4598 /*
4599 * Guest TR.
4600 */
4601 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR))
4602 {
4603 /*
4604 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is achieved
4605 * using the interrupt redirection bitmap (all bits cleared to let the guest handle INT-n's) in the TSS.
4606 * See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4607 */
4608 uint16_t u16Sel = 0;
4609 uint32_t u32Limit = 0;
4610 uint64_t u64Base = 0;
4611 uint32_t u32AccessRights = 0;
4612
4613 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4614 {
4615 u16Sel = pMixedCtx->tr.Sel;
4616 u32Limit = pMixedCtx->tr.u32Limit;
4617 u64Base = pMixedCtx->tr.u64Base;
4618 u32AccessRights = pMixedCtx->tr.Attr.u;
4619 }
4620 else
4621 {
4622 Assert(pVM->hm.s.vmx.pRealModeTSS);
4623 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4624
4625 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4626 RTGCPHYS GCPhys;
4627 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4628 AssertRCReturn(rc, rc);
4629
4630 X86DESCATTR DescAttr;
4631 DescAttr.u = 0;
4632 DescAttr.n.u1Present = 1;
4633 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4634
4635 u16Sel = 0;
4636 u32Limit = HM_VTX_TSS_SIZE;
4637 u64Base = GCPhys; /* in real-mode phys = virt. */
4638 u32AccessRights = DescAttr.u;
4639 }
4640
4641 /* Validate. */
4642 Assert(!(u16Sel & RT_BIT(2)));
4643 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4644 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4645 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4646 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4647 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4648 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4649 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4650 Assert( (u32Limit & 0xfff) == 0xfff
4651 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4652 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4653 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4654
4655 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4656 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4657 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4658 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4659 AssertRCReturn(rc, rc);
4660
4661 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_TR);
4662 Log4(("Load[%RU32]: VMX_VMCS_GUEST_TR_BASE=%#RX64\n", pVCpu->idCpu, u64Base));
4663 }
4664
4665 /*
4666 * Guest GDTR.
4667 */
4668 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR))
4669 {
4670 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4671 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4672 AssertRCReturn(rc, rc);
4673
4674 /* Validate. */
4675 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4676
4677 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_GDTR);
4678 Log4(("Load[%RU32]: VMX_VMCS_GUEST_GDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->gdtr.pGdt));
4679 }
4680
4681 /*
4682 * Guest LDTR.
4683 */
4684 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR))
4685 {
4686 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4687 uint32_t u32Access = 0;
4688 if (!pMixedCtx->ldtr.Attr.u)
4689 u32Access = X86DESCATTR_UNUSABLE;
4690 else
4691 u32Access = pMixedCtx->ldtr.Attr.u;
4692
4693 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4694 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4695 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4696 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4697 AssertRCReturn(rc, rc);
4698
4699 /* Validate. */
4700 if (!(u32Access & X86DESCATTR_UNUSABLE))
4701 {
4702 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4703 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4704 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4705 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4706 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4707 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4708 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4709 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4710 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4711 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4712 }
4713
4714 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_LDTR);
4715 Log4(("Load[%RU32]: VMX_VMCS_GUEST_LDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->ldtr.u64Base));
4716 }
4717
4718 /*
4719 * Guest IDTR.
4720 */
4721 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR))
4722 {
4723 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4724 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4725 AssertRCReturn(rc, rc);
4726
4727 /* Validate. */
4728 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4729
4730 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_IDTR);
4731 Log4(("Load[%RU32]: VMX_VMCS_GUEST_IDTR_BASE=%#RX64\n", pVCpu->idCpu, pMixedCtx->idtr.pIdt));
4732 }
4733
4734 return VINF_SUCCESS;
4735}
4736
4737
4738/**
4739 * Loads certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4740 * areas.
4741 *
4742 * These MSRs will automatically be loaded to the host CPU on every successful
4743 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4744 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4745 * -not- updated here for performance reasons. See hmR0VmxSaveHostMsrs().
4746 *
4747 * Also loads the sysenter MSRs into the guest-state area in the VMCS.
4748 *
4749 * @returns VBox status code.
4750 * @param pVCpu The cross context virtual CPU structure.
4751 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4752 * out-of-sync. Make sure to update the required fields
4753 * before using them.
4754 *
4755 * @remarks No-long-jump zone!!!
4756 */
4757static int hmR0VmxLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4758{
4759 AssertPtr(pVCpu);
4760 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4761
4762 /*
4763 * MSRs that we use the auto-load/store MSR area in the VMCS.
4764 */
4765 PVM pVM = pVCpu->CTX_SUFF(pVM);
4766 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS))
4767 {
4768 /* For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs(). */
4769#if HC_ARCH_BITS == 32
4770 if (pVM->hm.s.fAllow64BitGuests)
4771 {
4772 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4773 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4774 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4775 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4776 AssertRCReturn(rc, rc);
4777# ifdef LOG_ENABLED
4778 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4779 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4780 {
4781 Log4(("Load[%RU32]: MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", pVCpu->idCpu, i, pMsr->u32Msr,
4782 pMsr->u64Value));
4783 }
4784# endif
4785 }
4786#endif
4787 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4788 }
4789
4790 /*
4791 * Guest Sysenter MSRs.
4792 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4793 * VM-exits on WRMSRs for these MSRs.
4794 */
4795 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR))
4796 {
4797 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs); AssertRCReturn(rc, rc);
4798 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4799 }
4800
4801 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR))
4802 {
4803 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip); AssertRCReturn(rc, rc);
4804 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4805 }
4806
4807 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR))
4808 {
4809 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp); AssertRCReturn(rc, rc);
4810 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4811 }
4812
4813 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_EFER_MSR))
4814 {
4815 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4816 {
4817 /*
4818 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4819 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4820 */
4821 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4822 {
4823 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4824 AssertRCReturn(rc,rc);
4825 Log4(("Load[%RU32]: VMX_VMCS64_GUEST_EFER_FULL=%#RX64\n", pVCpu->idCpu, pMixedCtx->msrEFER));
4826 }
4827 else
4828 {
4829 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4830 NULL /* pfAddedAndUpdated */);
4831 AssertRCReturn(rc, rc);
4832
4833 /* We need to intercept reads too, see @bugref{7386#c16}. */
4834 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4835 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4836 Log4(("Load[%RU32]: MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", pVCpu->idCpu, MSR_K6_EFER,
4837 pMixedCtx->msrEFER, pVCpu->hm.s.vmx.cMsrs));
4838 }
4839 }
4840 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4841 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4842 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_EFER_MSR);
4843 }
4844
4845 return VINF_SUCCESS;
4846}
4847
4848
4849/**
4850 * Loads the guest activity state into the guest-state area in the VMCS.
4851 *
4852 * @returns VBox status code.
4853 * @param pVCpu The cross context virtual CPU structure.
4854 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4855 * out-of-sync. Make sure to update the required fields
4856 * before using them.
4857 *
4858 * @remarks No-long-jump zone!!!
4859 */
4860static int hmR0VmxLoadGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4861{
4862 NOREF(pMixedCtx);
4863 /** @todo See if we can make use of other states, e.g.
4864 * VMX_VMCS_GUEST_ACTIVITY_SHUTDOWN or HLT. */
4865 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE))
4866 {
4867 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, VMX_VMCS_GUEST_ACTIVITY_ACTIVE);
4868 AssertRCReturn(rc, rc);
4869
4870 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMX_GUEST_ACTIVITY_STATE);
4871 }
4872 return VINF_SUCCESS;
4873}
4874
4875
4876#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4877/**
4878 * Check if guest state allows safe use of 32-bit switcher again.
4879 *
4880 * Segment bases and protected mode structures must be 32-bit addressable
4881 * because the 32-bit switcher will ignore high dword when writing these VMCS
4882 * fields. See @bugref{8432} for details.
4883 *
4884 * @returns true if safe, false if must continue to use the 64-bit switcher.
4885 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4886 * out-of-sync. Make sure to update the required fields
4887 * before using them.
4888 *
4889 * @remarks No-long-jump zone!!!
4890 */
4891static bool hmR0VmxIs32BitSwitcherSafe(PCPUMCTX pMixedCtx)
4892{
4893 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000))
4894 return false;
4895 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000))
4896 return false;
4897 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000))
4898 return false;
4899 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000))
4900 return false;
4901 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000))
4902 return false;
4903 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000))
4904 return false;
4905 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000))
4906 return false;
4907 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000))
4908 return false;
4909 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000))
4910 return false;
4911 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000))
4912 return false;
4913 /* All good, bases are 32-bit. */
4914 return true;
4915}
4916#endif
4917
4918
4919/**
4920 * Sets up the appropriate function to run guest code.
4921 *
4922 * @returns VBox status code.
4923 * @param pVCpu The cross context virtual CPU structure.
4924 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4925 * out-of-sync. Make sure to update the required fields
4926 * before using them.
4927 *
4928 * @remarks No-long-jump zone!!!
4929 */
4930static int hmR0VmxSetupVMRunHandler(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4931{
4932 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4933 {
4934#ifndef VBOX_ENABLE_64_BITS_GUESTS
4935 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4936#endif
4937 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4938#if HC_ARCH_BITS == 32
4939 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4940 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4941 {
4942 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4943 {
4944 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4945 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4946 | HM_CHANGED_VMX_ENTRY_CTLS
4947 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4948 }
4949 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4950
4951 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4952 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4953 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4954 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 64-bit switcher\n", pVCpu->idCpu));
4955 }
4956#else
4957 /* 64-bit host. */
4958 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4959#endif
4960 }
4961 else
4962 {
4963 /* Guest is not in long mode, use the 32-bit handler. */
4964#if HC_ARCH_BITS == 32
4965 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4966 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4967 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4968 {
4969 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4970 AssertMsg(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_VMX_EXIT_CTLS
4971 | HM_CHANGED_VMX_ENTRY_CTLS
4972 | HM_CHANGED_GUEST_EFER_MSR), ("flags=%#x\n", HMCPU_CF_VALUE(pVCpu)));
4973 }
4974# ifdef VBOX_ENABLE_64_BITS_GUESTS
4975 /*
4976 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel design, see @bugref{8432#c7}.
4977 * If real-on-v86 mode is active, clear the 64-bit switcher flag because now we know the guest is in a sane
4978 * state where it's safe to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4979 * the much faster 32-bit switcher again.
4980 */
4981 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4982 {
4983 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4984 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher\n", pVCpu->idCpu));
4985 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4986 }
4987 else
4988 {
4989 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4990 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4991 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4992 {
4993 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4994 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4995 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR
4996 | HM_CHANGED_VMX_ENTRY_CTLS
4997 | HM_CHANGED_VMX_EXIT_CTLS
4998 | HM_CHANGED_HOST_CONTEXT);
4999 Log4(("Load[%RU32]: hmR0VmxSetupVMRunHandler: selected 32-bit switcher (safe)\n", pVCpu->idCpu));
5000 }
5001 }
5002# else
5003 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5004# endif
5005#else
5006 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
5007#endif
5008 }
5009 Assert(pVCpu->hm.s.vmx.pfnStartVM);
5010 return VINF_SUCCESS;
5011}
5012
5013
5014/**
5015 * Wrapper for running the guest code in VT-x.
5016 *
5017 * @returns VBox status code, no informational status codes.
5018 * @param pVM The cross context VM structure.
5019 * @param pVCpu The cross context virtual CPU structure.
5020 * @param pCtx Pointer to the guest-CPU context.
5021 *
5022 * @remarks No-long-jump zone!!!
5023 */
5024DECLINLINE(int) hmR0VmxRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
5025{
5026 /*
5027 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses floating-point operations
5028 * using SSE instructions. Some XMM registers (XMM6-XMM15) are callee-saved and thus the need for this XMM wrapper.
5029 * Refer MSDN docs. "Configuring Programs for 64-bit / x64 Software Conventions / Register Usage" for details.
5030 */
5031 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5032 /** @todo Add stats for resume vs launch. */
5033#ifdef VBOX_WITH_KERNEL_USING_XMM
5034 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5035#else
5036 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5037#endif
5038 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5039 return rc;
5040}
5041
5042
5043/**
5044 * Reports world-switch error and dumps some useful debug info.
5045 *
5046 * @param pVM The cross context VM structure.
5047 * @param pVCpu The cross context virtual CPU structure.
5048 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5049 * @param pCtx Pointer to the guest-CPU context.
5050 * @param pVmxTransient Pointer to the VMX transient structure (only
5051 * exitReason updated).
5052 */
5053static void hmR0VmxReportWorldSwitchError(PVM pVM, PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5054{
5055 Assert(pVM);
5056 Assert(pVCpu);
5057 Assert(pCtx);
5058 Assert(pVmxTransient);
5059 HMVMX_ASSERT_PREEMPT_SAFE();
5060
5061 Log4(("VM-entry failure: %Rrc\n", rcVMRun));
5062 switch (rcVMRun)
5063 {
5064 case VERR_VMX_INVALID_VMXON_PTR:
5065 AssertFailed();
5066 break;
5067 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5068 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5069 {
5070 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5071 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5072 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5073 AssertRC(rc);
5074
5075 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5076 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5077 Cannot do it here as we may have been long preempted. */
5078
5079#ifdef VBOX_STRICT
5080 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5081 pVmxTransient->uExitReason));
5082 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5083 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5084 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5085 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5086 else
5087 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5088 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5089 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5090
5091 /* VMX control bits. */
5092 uint32_t u32Val;
5093 uint64_t u64Val;
5094 RTHCUINTREG uHCReg;
5095 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5096 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5097 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5098 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5099 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5100 {
5101 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5102 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5103 }
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5107 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5110 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5112 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5113 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5114 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5115 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5116 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5117 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5118 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5119 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5120 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5121 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5122 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5123 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5124 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5125 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5126 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5127 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5128 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5129 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5130 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5131 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5132 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5133 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5134 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5135 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5136 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5137 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5138 if (pVM->hm.s.fNestedPaging)
5139 {
5140 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5141 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5142 }
5143
5144 /* Guest bits. */
5145 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5146 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5147 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5148 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5149 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5150 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5151 if (pVM->hm.s.vmx.fVpid)
5152 {
5153 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5154 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5155 }
5156
5157 /* Host bits. */
5158 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5159 Log4(("Host CR0 %#RHr\n", uHCReg));
5160 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5161 Log4(("Host CR3 %#RHr\n", uHCReg));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5163 Log4(("Host CR4 %#RHr\n", uHCReg));
5164
5165 RTGDTR HostGdtr;
5166 PCX86DESCHC pDesc;
5167 ASMGetGDTR(&HostGdtr);
5168 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5169 Log4(("Host CS %#08x\n", u32Val));
5170 if (u32Val < HostGdtr.cbGdt)
5171 {
5172 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5173 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5174 }
5175
5176 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5177 Log4(("Host DS %#08x\n", u32Val));
5178 if (u32Val < HostGdtr.cbGdt)
5179 {
5180 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5181 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5182 }
5183
5184 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5185 Log4(("Host ES %#08x\n", u32Val));
5186 if (u32Val < HostGdtr.cbGdt)
5187 {
5188 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5189 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5190 }
5191
5192 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5193 Log4(("Host FS %#08x\n", u32Val));
5194 if (u32Val < HostGdtr.cbGdt)
5195 {
5196 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5197 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5198 }
5199
5200 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5201 Log4(("Host GS %#08x\n", u32Val));
5202 if (u32Val < HostGdtr.cbGdt)
5203 {
5204 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5205 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5206 }
5207
5208 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5209 Log4(("Host SS %#08x\n", u32Val));
5210 if (u32Val < HostGdtr.cbGdt)
5211 {
5212 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5213 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5214 }
5215
5216 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5217 Log4(("Host TR %#08x\n", u32Val));
5218 if (u32Val < HostGdtr.cbGdt)
5219 {
5220 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5221 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5222 }
5223
5224 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5225 Log4(("Host TR Base %#RHv\n", uHCReg));
5226 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5227 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5228 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5229 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5230 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5231 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5232 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5233 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5234 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5235 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5236 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5237 Log4(("Host RSP %#RHv\n", uHCReg));
5238 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5239 Log4(("Host RIP %#RHv\n", uHCReg));
5240# if HC_ARCH_BITS == 64
5241 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5242 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5243 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5244 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5245 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5246 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5247# endif
5248#endif /* VBOX_STRICT */
5249 break;
5250 }
5251
5252 default:
5253 /* Impossible */
5254 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5255 break;
5256 }
5257 NOREF(pVM); NOREF(pCtx);
5258}
5259
5260
5261#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5262#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5263# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5264#endif
5265#ifdef VBOX_STRICT
5266static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5267{
5268 switch (idxField)
5269 {
5270 case VMX_VMCS_GUEST_RIP:
5271 case VMX_VMCS_GUEST_RSP:
5272 case VMX_VMCS_GUEST_SYSENTER_EIP:
5273 case VMX_VMCS_GUEST_SYSENTER_ESP:
5274 case VMX_VMCS_GUEST_GDTR_BASE:
5275 case VMX_VMCS_GUEST_IDTR_BASE:
5276 case VMX_VMCS_GUEST_CS_BASE:
5277 case VMX_VMCS_GUEST_DS_BASE:
5278 case VMX_VMCS_GUEST_ES_BASE:
5279 case VMX_VMCS_GUEST_FS_BASE:
5280 case VMX_VMCS_GUEST_GS_BASE:
5281 case VMX_VMCS_GUEST_SS_BASE:
5282 case VMX_VMCS_GUEST_LDTR_BASE:
5283 case VMX_VMCS_GUEST_TR_BASE:
5284 case VMX_VMCS_GUEST_CR3:
5285 return true;
5286 }
5287 return false;
5288}
5289
5290static bool hmR0VmxIsValidReadField(uint32_t idxField)
5291{
5292 switch (idxField)
5293 {
5294 /* Read-only fields. */
5295 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5296 return true;
5297 }
5298 /* Remaining readable fields should also be writable. */
5299 return hmR0VmxIsValidWriteField(idxField);
5300}
5301#endif /* VBOX_STRICT */
5302
5303
5304/**
5305 * Executes the specified handler in 64-bit mode.
5306 *
5307 * @returns VBox status code (no informational status codes).
5308 * @param pVM The cross context VM structure.
5309 * @param pVCpu The cross context virtual CPU structure.
5310 * @param pCtx Pointer to the guest CPU context.
5311 * @param enmOp The operation to perform.
5312 * @param cParams Number of parameters.
5313 * @param paParam Array of 32-bit parameters.
5314 */
5315VMMR0DECL(int) VMXR0Execute64BitsHandler(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx, HM64ON32OP enmOp,
5316 uint32_t cParams, uint32_t *paParam)
5317{
5318 NOREF(pCtx);
5319
5320 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5321 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5322 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5323 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5324
5325#ifdef VBOX_STRICT
5326 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5327 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5328
5329 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5330 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5331#endif
5332
5333 /* Disable interrupts. */
5334 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5335
5336#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5337 RTCPUID idHostCpu = RTMpCpuId();
5338 CPUMR0SetLApic(pVCpu, idHostCpu);
5339#endif
5340
5341 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5342 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5343
5344 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5345 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5346 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5347
5348 /* Leave VMX Root Mode. */
5349 VMXDisable();
5350
5351 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5352
5353 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5354 CPUMSetHyperEIP(pVCpu, enmOp);
5355 for (int i = (int)cParams - 1; i >= 0; i--)
5356 CPUMPushHyper(pVCpu, paParam[i]);
5357
5358 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5359
5360 /* Call the switcher. */
5361 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5362 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5363
5364 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5365 /* Make sure the VMX instructions don't cause #UD faults. */
5366 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5367
5368 /* Re-enter VMX Root Mode */
5369 int rc2 = VMXEnable(HCPhysCpuPage);
5370 if (RT_FAILURE(rc2))
5371 {
5372 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5373 ASMSetFlags(fOldEFlags);
5374 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5375 return rc2;
5376 }
5377
5378 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5379 AssertRC(rc2);
5380 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5381 Assert(!(ASMGetFlags() & X86_EFL_IF));
5382 ASMSetFlags(fOldEFlags);
5383 return rc;
5384}
5385
5386
5387/**
5388 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5389 * supporting 64-bit guests.
5390 *
5391 * @returns VBox status code.
5392 * @param fResume Whether to VMLAUNCH or VMRESUME.
5393 * @param pCtx Pointer to the guest-CPU context.
5394 * @param pCache Pointer to the VMCS cache.
5395 * @param pVM The cross context VM structure.
5396 * @param pVCpu The cross context virtual CPU structure.
5397 */
5398DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5399{
5400 NOREF(fResume);
5401
5402 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5403 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5404
5405#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5406 pCache->uPos = 1;
5407 pCache->interPD = PGMGetInterPaeCR3(pVM);
5408 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5409#endif
5410
5411#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5412 pCache->TestIn.HCPhysCpuPage = 0;
5413 pCache->TestIn.HCPhysVmcs = 0;
5414 pCache->TestIn.pCache = 0;
5415 pCache->TestOut.HCPhysVmcs = 0;
5416 pCache->TestOut.pCache = 0;
5417 pCache->TestOut.pCtx = 0;
5418 pCache->TestOut.eflags = 0;
5419#else
5420 NOREF(pCache);
5421#endif
5422
5423 uint32_t aParam[10];
5424 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5425 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5426 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5427 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5428 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5429 aParam[5] = 0;
5430 aParam[6] = VM_RC_ADDR(pVM, pVM);
5431 aParam[7] = 0;
5432 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5433 aParam[9] = 0;
5434
5435#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5436 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5437 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5438#endif
5439 int rc = VMXR0Execute64BitsHandler(pVM, pVCpu, pCtx, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5440
5441#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5442 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5443 Assert(pCtx->dr[4] == 10);
5444 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5445#endif
5446
5447#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5448 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5449 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5450 pVCpu->hm.s.vmx.HCPhysVmcs));
5451 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5452 pCache->TestOut.HCPhysVmcs));
5453 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5454 pCache->TestOut.pCache));
5455 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5456 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5457 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5458 pCache->TestOut.pCtx));
5459 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5460#endif
5461 return rc;
5462}
5463
5464
5465/**
5466 * Initialize the VMCS-Read cache.
5467 *
5468 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5469 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5470 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5471 * (those that have a 32-bit FULL & HIGH part).
5472 *
5473 * @returns VBox status code.
5474 * @param pVM The cross context VM structure.
5475 * @param pVCpu The cross context virtual CPU structure.
5476 */
5477static int hmR0VmxInitVmcsReadCache(PVM pVM, PVMCPU pVCpu)
5478{
5479#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5480{ \
5481 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5482 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5483 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5484 ++cReadFields; \
5485}
5486
5487 AssertPtr(pVM);
5488 AssertPtr(pVCpu);
5489 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5490 uint32_t cReadFields = 0;
5491
5492 /*
5493 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5494 * and serve to indicate exceptions to the rules.
5495 */
5496
5497 /* Guest-natural selector base fields. */
5498#if 0
5499 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5502#endif
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5509 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5510 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5511 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5513 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5514 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5515#if 0
5516 /* Unused natural width guest-state fields. */
5517 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5518 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5519#endif
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5521 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5522
5523 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for these 64-bit fields (using "FULL" and "HIGH" fields). */
5524#if 0
5525 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5526 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5527 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5528 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5529 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5530 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5531 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5532 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5533 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5534#endif
5535
5536 /* Natural width guest-state fields. */
5537 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5538#if 0
5539 /* Currently unused field. */
5540 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5541#endif
5542
5543 if (pVM->hm.s.fNestedPaging)
5544 {
5545 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5546 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5547 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5548 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5549 }
5550 else
5551 {
5552 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5553 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5554 }
5555
5556#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5557 return VINF_SUCCESS;
5558}
5559
5560
5561/**
5562 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5563 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5564 * darwin, running 64-bit guests).
5565 *
5566 * @returns VBox status code.
5567 * @param pVCpu The cross context virtual CPU structure.
5568 * @param idxField The VMCS field encoding.
5569 * @param u64Val 16, 32 or 64-bit value.
5570 */
5571VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5572{
5573 int rc;
5574 switch (idxField)
5575 {
5576 /*
5577 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5578 */
5579 /* 64-bit Control fields. */
5580 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5581 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5582 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5583 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5584 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5585 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5586 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5587 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5588 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5589 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5590 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5591 case VMX_VMCS64_CTRL_EPTP_FULL:
5592 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5593 /* 64-bit Guest-state fields. */
5594 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5595 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5596 case VMX_VMCS64_GUEST_PAT_FULL:
5597 case VMX_VMCS64_GUEST_EFER_FULL:
5598 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5599 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5600 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5601 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5602 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5603 /* 64-bit Host-state fields. */
5604 case VMX_VMCS64_HOST_PAT_FULL:
5605 case VMX_VMCS64_HOST_EFER_FULL:
5606 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5607 {
5608 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5609 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5610 break;
5611 }
5612
5613 /*
5614 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5615 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5616 */
5617 /* Natural-width Guest-state fields. */
5618 case VMX_VMCS_GUEST_CR3:
5619 case VMX_VMCS_GUEST_ES_BASE:
5620 case VMX_VMCS_GUEST_CS_BASE:
5621 case VMX_VMCS_GUEST_SS_BASE:
5622 case VMX_VMCS_GUEST_DS_BASE:
5623 case VMX_VMCS_GUEST_FS_BASE:
5624 case VMX_VMCS_GUEST_GS_BASE:
5625 case VMX_VMCS_GUEST_LDTR_BASE:
5626 case VMX_VMCS_GUEST_TR_BASE:
5627 case VMX_VMCS_GUEST_GDTR_BASE:
5628 case VMX_VMCS_GUEST_IDTR_BASE:
5629 case VMX_VMCS_GUEST_RSP:
5630 case VMX_VMCS_GUEST_RIP:
5631 case VMX_VMCS_GUEST_SYSENTER_ESP:
5632 case VMX_VMCS_GUEST_SYSENTER_EIP:
5633 {
5634 if (!(RT_HI_U32(u64Val)))
5635 {
5636 /* If this field is 64-bit, VT-x will zero out the top bits. */
5637 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5638 }
5639 else
5640 {
5641 /* Assert that only the 32->64 switcher case should ever come here. */
5642 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5643 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5644 }
5645 break;
5646 }
5647
5648 default:
5649 {
5650 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5651 rc = VERR_INVALID_PARAMETER;
5652 break;
5653 }
5654 }
5655 AssertRCReturn(rc, rc);
5656 return rc;
5657}
5658
5659
5660/**
5661 * Queue up a VMWRITE by using the VMCS write cache.
5662 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5663 *
5664 * @param pVCpu The cross context virtual CPU structure.
5665 * @param idxField The VMCS field encoding.
5666 * @param u64Val 16, 32 or 64-bit value.
5667 */
5668VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5669{
5670 AssertPtr(pVCpu);
5671 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5672
5673 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5674 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5675
5676 /* Make sure there are no duplicates. */
5677 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5678 {
5679 if (pCache->Write.aField[i] == idxField)
5680 {
5681 pCache->Write.aFieldVal[i] = u64Val;
5682 return VINF_SUCCESS;
5683 }
5684 }
5685
5686 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5687 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5688 pCache->Write.cValidEntries++;
5689 return VINF_SUCCESS;
5690}
5691#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5692
5693
5694/**
5695 * Sets up the usage of TSC-offsetting and updates the VMCS.
5696 *
5697 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5698 * VMX preemption timer.
5699 *
5700 * @returns VBox status code.
5701 * @param pVM The cross context VM structure.
5702 * @param pVCpu The cross context virtual CPU structure.
5703 *
5704 * @remarks No-long-jump zone!!!
5705 */
5706static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVM pVM, PVMCPU pVCpu)
5707{
5708 int rc;
5709 bool fOffsettedTsc;
5710 bool fParavirtTsc;
5711 if (pVM->hm.s.vmx.fUsePreemptTimer)
5712 {
5713 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset,
5714 &fOffsettedTsc, &fParavirtTsc);
5715
5716 /* Make sure the returned values have sane upper and lower boundaries. */
5717 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5718 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5719 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5720 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5721
5722 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5723 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount); AssertRC(rc);
5724 }
5725 else
5726 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &pVCpu->hm.s.vmx.u64TSCOffset, &fParavirtTsc);
5727
5728 /** @todo later optimize this to be done elsewhere and not before every
5729 * VM-entry. */
5730 if (fParavirtTsc)
5731 {
5732 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5733 information before every VM-entry, hence disable it for performance sake. */
5734#if 0
5735 rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5736 AssertRC(rc);
5737#endif
5738 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5739 }
5740
5741 if (fOffsettedTsc && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5742 {
5743 /* Note: VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT takes precedence over TSC_OFFSET, applies to RDTSCP too. */
5744 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TSCOffset); AssertRC(rc);
5745
5746 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5747 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5748 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5749 }
5750 else
5751 {
5752 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5753 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5754 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls); AssertRC(rc);
5755 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5756 }
5757}
5758
5759
5760#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5761/**
5762 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5763 * VM-exit interruption info type.
5764 *
5765 * @returns The IEM exception flags.
5766 * @param uVector The event vector.
5767 * @param uVmxVectorType The VMX event type.
5768 *
5769 * @remarks This function currently only constructs flags required for
5770 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5771 * and CR2 aspects of an exception are not included).
5772 */
5773static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5774{
5775 uint32_t fIemXcptFlags;
5776 switch (uVmxVectorType)
5777 {
5778 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5779 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5780 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5781 break;
5782
5783 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5784 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5785 break;
5786
5787 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5788 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5789 break;
5790
5791 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5792 {
5793 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5794 if (uVector == X86_XCPT_BP)
5795 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5796 else if (uVector == X86_XCPT_OF)
5797 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5798 else
5799 {
5800 fIemXcptFlags = 0;
5801 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5802 }
5803 break;
5804 }
5805
5806 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5807 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5808 break;
5809
5810 default:
5811 fIemXcptFlags = 0;
5812 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5813 break;
5814 }
5815 return fIemXcptFlags;
5816}
5817
5818#else
5819/**
5820 * Determines if an exception is a contributory exception.
5821 *
5822 * Contributory exceptions are ones which can cause double-faults unless the
5823 * original exception was a benign exception. Page-fault is intentionally not
5824 * included here as it's a conditional contributory exception.
5825 *
5826 * @returns true if the exception is contributory, false otherwise.
5827 * @param uVector The exception vector.
5828 */
5829DECLINLINE(bool) hmR0VmxIsContributoryXcpt(const uint32_t uVector)
5830{
5831 switch (uVector)
5832 {
5833 case X86_XCPT_GP:
5834 case X86_XCPT_SS:
5835 case X86_XCPT_NP:
5836 case X86_XCPT_TS:
5837 case X86_XCPT_DE:
5838 return true;
5839 default:
5840 break;
5841 }
5842 return false;
5843}
5844#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
5845
5846
5847/**
5848 * Sets an event as a pending event to be injected into the guest.
5849 *
5850 * @param pVCpu The cross context virtual CPU structure.
5851 * @param u32IntInfo The VM-entry interruption-information field.
5852 * @param cbInstr The VM-entry instruction length in bytes (for software
5853 * interrupts, exceptions and privileged software
5854 * exceptions).
5855 * @param u32ErrCode The VM-entry exception error code.
5856 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5857 * page-fault.
5858 *
5859 * @remarks Statistics counter assumes this is a guest event being injected or
5860 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5861 * always incremented.
5862 */
5863DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5864 RTGCUINTPTR GCPtrFaultAddress)
5865{
5866 Assert(!pVCpu->hm.s.Event.fPending);
5867 pVCpu->hm.s.Event.fPending = true;
5868 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5869 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5870 pVCpu->hm.s.Event.cbInstr = cbInstr;
5871 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5872}
5873
5874
5875/**
5876 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5877 *
5878 * @param pVCpu The cross context virtual CPU structure.
5879 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5880 * out-of-sync. Make sure to update the required fields
5881 * before using them.
5882 */
5883DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5884{
5885 NOREF(pMixedCtx);
5886 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5887 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5888 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5889 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5890}
5891
5892
5893/**
5894 * Handle a condition that occurred while delivering an event through the guest
5895 * IDT.
5896 *
5897 * @returns Strict VBox status code (i.e. informational status codes too).
5898 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5899 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5900 * to continue execution of the guest which will delivery the \#DF.
5901 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5902 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5903 *
5904 * @param pVCpu The cross context virtual CPU structure.
5905 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5906 * out-of-sync. Make sure to update the required fields
5907 * before using them.
5908 * @param pVmxTransient Pointer to the VMX transient structure.
5909 *
5910 * @remarks No-long-jump zone!!!
5911 */
5912static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5913{
5914 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5915
5916 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5917 rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient); AssertRCReturn(rc2, rc2);
5918
5919 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5920 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5921 {
5922 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5923 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5924#ifdef HMVMX_USE_IEM_EVENT_REFLECTION
5925 /*
5926 * If the event was a software interrupt (generated with INT n) or a software exception (generated
5927 * by INT3/INTO) or a privileged software exception (generated by INT1), we can handle the VM-exit
5928 * and continue guest execution which will re-execute the instruction rather than re-injecting the
5929 * exception, as that can cause premature trips to ring-3 before injection and involve TRPM which
5930 * currently has no way of storing that these exceptions were caused by these instructions
5931 * (ICEBP's #DB poses the problem).
5932 */
5933 IEMXCPTRAISE enmRaise;
5934 IEMXCPTRAISEINFO fRaiseInfo;
5935 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5936 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5937 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5938 {
5939 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5940 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5941 }
5942 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5943 {
5944 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5945 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5946 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5947 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5948 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5949 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5950 uExitVectorType), VERR_VMX_IPE_5);
5951 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5952
5953 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5954 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5955 {
5956 pVmxTransient->fVectoringPF = true;
5957 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5958 }
5959 }
5960 else
5961 {
5962 /*
5963 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5964 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5965 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5966 */
5967 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5968 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5969 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5970 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5971 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5972 }
5973
5974 /*
5975 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5976 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5977 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5978 * subsequent VM-entry would fail.
5979 *
5980 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5981 */
5982 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5983 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5984 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5985 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5986 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5987 {
5988 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5989 }
5990
5991 switch (enmRaise)
5992 {
5993 case IEMXCPTRAISE_CURRENT_XCPT:
5994 {
5995 Log4(("IDT: vcpu[%RU32] Pending secondary xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n", pVCpu->idCpu,
5996 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5997 Assert(rcStrict == VINF_SUCCESS);
5998 break;
5999 }
6000
6001 case IEMXCPTRAISE_PREV_EVENT:
6002 {
6003 uint32_t u32ErrCode;
6004 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6005 {
6006 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6007 AssertRCReturn(rc2, rc2);
6008 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6009 }
6010 else
6011 u32ErrCode = 0;
6012
6013 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
6014 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6015 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6016 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6017
6018 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6019 pVCpu->hm.s.Event.u32ErrCode));
6020 Assert(rcStrict == VINF_SUCCESS);
6021 break;
6022 }
6023
6024 case IEMXCPTRAISE_REEXEC_INSTR:
6025 Assert(rcStrict == VINF_SUCCESS);
6026 break;
6027
6028 case IEMXCPTRAISE_DOUBLE_FAULT:
6029 {
6030 /*
6031 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
6032 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
6033 */
6034 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6035 {
6036 pVmxTransient->fVectoringDoublePF = true;
6037 Log4(("IDT: vcpu[%RU32] Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
6038 pMixedCtx->cr2));
6039 rcStrict = VINF_SUCCESS;
6040 }
6041 else
6042 {
6043 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6044 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6045 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6046 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6047 rcStrict = VINF_HM_DOUBLE_FAULT;
6048 }
6049 break;
6050 }
6051
6052 case IEMXCPTRAISE_TRIPLE_FAULT:
6053 {
6054 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6055 uExitVector));
6056 rcStrict = VINF_EM_RESET;
6057 break;
6058 }
6059
6060 case IEMXCPTRAISE_CPU_HANG:
6061 {
6062 Log4(("IDT: vcpu[%RU32] Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", pVCpu->idCpu, fRaiseInfo));
6063 rcStrict = VERR_EM_GUEST_CPU_HANG;
6064 break;
6065 }
6066
6067 default:
6068 {
6069 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6070 rcStrict = VERR_VMX_IPE_2;
6071 break;
6072 }
6073 }
6074#else
6075 typedef enum
6076 {
6077 VMXREFLECTXCPT_XCPT, /* Reflect the exception to the guest or for further evaluation by VMM. */
6078 VMXREFLECTXCPT_DF, /* Reflect the exception as a double-fault to the guest. */
6079 VMXREFLECTXCPT_TF, /* Indicate a triple faulted state to the VMM. */
6080 VMXREFLECTXCPT_HANG, /* Indicate bad VM trying to deadlock the CPU. */
6081 VMXREFLECTXCPT_NONE /* Nothing to reflect. */
6082 } VMXREFLECTXCPT;
6083
6084 /* See Intel spec. 30.7.1.1 "Reflecting Exceptions to Guest Software". */
6085 VMXREFLECTXCPT enmReflect = VMXREFLECTXCPT_NONE;
6086 if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
6087 {
6088 if (uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT)
6089 {
6090 enmReflect = VMXREFLECTXCPT_XCPT;
6091#ifdef VBOX_STRICT
6092 if ( hmR0VmxIsContributoryXcpt(uIdtVector)
6093 && uExitVector == X86_XCPT_PF)
6094 {
6095 Log4(("IDT: vcpu[%RU32] Contributory #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6096 }
6097#endif
6098 if ( uExitVector == X86_XCPT_PF
6099 && uIdtVector == X86_XCPT_PF)
6100 {
6101 pVmxTransient->fVectoringDoublePF = true;
6102 Log4(("IDT: vcpu[%RU32] Vectoring Double #PF uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6103 }
6104 else if ( uExitVector == X86_XCPT_AC
6105 && uIdtVector == X86_XCPT_AC)
6106 {
6107 enmReflect = VMXREFLECTXCPT_HANG;
6108 Log4(("IDT: Nested #AC - Bad guest\n"));
6109 }
6110 else if ( (pVCpu->hm.s.vmx.u32XcptBitmap & HMVMX_CONTRIBUTORY_XCPT_MASK)
6111 && hmR0VmxIsContributoryXcpt(uExitVector)
6112 && ( hmR0VmxIsContributoryXcpt(uIdtVector)
6113 || uIdtVector == X86_XCPT_PF))
6114 {
6115 enmReflect = VMXREFLECTXCPT_DF;
6116 }
6117 else if (uIdtVector == X86_XCPT_DF)
6118 enmReflect = VMXREFLECTXCPT_TF;
6119 }
6120 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6121 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6122 {
6123 /*
6124 * Ignore software interrupts (INT n), software exceptions (#BP, #OF) and
6125 * privileged software exception (#DB from ICEBP) as they reoccur when restarting the instruction.
6126 */
6127 enmReflect = VMXREFLECTXCPT_XCPT;
6128
6129 if (uExitVector == X86_XCPT_PF)
6130 {
6131 pVmxTransient->fVectoringPF = true;
6132 Log4(("IDT: vcpu[%RU32] Vectoring #PF due to Ext-Int/NMI. uCR2=%#RX64\n", pVCpu->idCpu, pMixedCtx->cr2));
6133 }
6134 }
6135 }
6136 else if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6137 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT
6138 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI)
6139 {
6140 /*
6141 * If event delivery caused an EPT violation/misconfig or APIC access VM-exit, then the VM-exit
6142 * interruption-information will not be valid as it's not an exception and we end up here. In such cases,
6143 * it is sufficient to reflect the original exception to the guest after handling the VM-exit.
6144 */
6145 enmReflect = VMXREFLECTXCPT_XCPT;
6146 }
6147
6148 /*
6149 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig etc.) occurred
6150 * while delivering the NMI, we need to clear the block-by-NMI field in the guest interruptibility-state before
6151 * re-delivering the NMI after handling the VM-exit. Otherwise the subsequent VM-entry would fail.
6152 *
6153 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
6154 */
6155 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
6156 && enmReflect == VMXREFLECTXCPT_XCPT
6157 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
6158 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6159 {
6160 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6161 }
6162
6163 switch (enmReflect)
6164 {
6165 case VMXREFLECTXCPT_XCPT:
6166 {
6167 Assert( uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6168 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6169 && uIdtVectorType != VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT);
6170
6171 uint32_t u32ErrCode = 0;
6172 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
6173 {
6174 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
6175 AssertRCReturn(rc2, rc2);
6176 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
6177 }
6178
6179 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF. See hmR0VmxExitXcptPF(). */
6180 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6181 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
6182 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
6183 rcStrict = VINF_SUCCESS;
6184 Log4(("IDT: vcpu[%RU32] Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->idCpu,
6185 pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.u32ErrCode));
6186
6187 break;
6188 }
6189
6190 case VMXREFLECTXCPT_DF:
6191 {
6192 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6193 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6194 rcStrict = VINF_HM_DOUBLE_FAULT;
6195 Log4(("IDT: vcpu[%RU32] Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->idCpu,
6196 pVCpu->hm.s.Event.u64IntInfo, uIdtVector, uExitVector));
6197
6198 break;
6199 }
6200
6201 case VMXREFLECTXCPT_TF:
6202 {
6203 rcStrict = VINF_EM_RESET;
6204 Log4(("IDT: vcpu[%RU32] Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", pVCpu->idCpu, uIdtVector,
6205 uExitVector));
6206 break;
6207 }
6208
6209 case VMXREFLECTXCPT_HANG:
6210 {
6211 rcStrict = VERR_EM_GUEST_CPU_HANG;
6212 break;
6213 }
6214
6215 default:
6216 Assert(rcStrict == VINF_SUCCESS);
6217 break;
6218 }
6219#endif /* HMVMX_USE_IEM_EVENT_REFLECTION */
6220 }
6221 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6222 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6223 && uExitVector != X86_XCPT_DF
6224 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6225 {
6226 /*
6227 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6228 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6229 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6230 */
6231 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6232 {
6233 Log4(("hmR0VmxCheckExitDueToEventDelivery: vcpu[%RU32] Setting VMCPU_FF_BLOCK_NMIS. Valid=%RTbool uExitReason=%u\n",
6234 pVCpu->idCpu, VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6235 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6236 }
6237 }
6238
6239 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6240 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6241 return rcStrict;
6242}
6243
6244
6245/**
6246 * Saves the guest's CR0 register from the VMCS into the guest-CPU context.
6247 *
6248 * @returns VBox status code.
6249 * @param pVCpu The cross context virtual CPU structure.
6250 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6251 * out-of-sync. Make sure to update the required fields
6252 * before using them.
6253 *
6254 * @remarks No-long-jump zone!!!
6255 */
6256static int hmR0VmxSaveGuestCR0(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6257{
6258 NOREF(pMixedCtx);
6259
6260 /*
6261 * While in the middle of saving guest-CR0, we could get preempted and re-invoked from the preemption hook,
6262 * see hmR0VmxLeave(). Safer to just make this code non-preemptible.
6263 */
6264 VMMRZCallRing3Disable(pVCpu);
6265 HM_DISABLE_PREEMPT();
6266
6267 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0))
6268 {
6269#ifndef DEBUG_bird /** @todo this triggers running bs3-cpu-generated-1.img with --debug-command-line
6270 * and 'dbgc-init' containing:
6271 * sxe "xcpt_de"
6272 * sxe "xcpt_bp"
6273 * sxi "xcpt_gp"
6274 * sxi "xcpt_ss"
6275 * sxi "xcpt_np"
6276 */
6277 /** @todo r=ramshankar: Should be fixed after r119291. */
6278 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0));
6279#endif
6280 uint32_t uVal = 0;
6281 uint32_t uShadow = 0;
6282 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uVal);
6283 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uShadow);
6284 AssertRCReturn(rc, rc);
6285
6286 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR0Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR0Mask);
6287 CPUMSetGuestCR0(pVCpu, uVal);
6288 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0);
6289 }
6290
6291 HM_RESTORE_PREEMPT();
6292 VMMRZCallRing3Enable(pVCpu);
6293 return VINF_SUCCESS;
6294}
6295
6296
6297/**
6298 * Saves the guest's CR4 register from the VMCS into the guest-CPU context.
6299 *
6300 * @returns VBox status code.
6301 * @param pVCpu The cross context virtual CPU structure.
6302 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6303 * out-of-sync. Make sure to update the required fields
6304 * before using them.
6305 *
6306 * @remarks No-long-jump zone!!!
6307 */
6308static int hmR0VmxSaveGuestCR4(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6309{
6310 NOREF(pMixedCtx);
6311
6312 int rc = VINF_SUCCESS;
6313 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4))
6314 {
6315 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR4));
6316 uint32_t uVal = 0;
6317 uint32_t uShadow = 0;
6318 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uVal);
6319 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uShadow);
6320 AssertRCReturn(rc, rc);
6321
6322 uVal = (uShadow & pVCpu->hm.s.vmx.u32CR4Mask) | (uVal & ~pVCpu->hm.s.vmx.u32CR4Mask);
6323 CPUMSetGuestCR4(pVCpu, uVal);
6324 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4);
6325 }
6326 return rc;
6327}
6328
6329
6330/**
6331 * Saves the guest's RIP register from the VMCS into the guest-CPU context.
6332 *
6333 * @returns VBox status code.
6334 * @param pVCpu The cross context virtual CPU structure.
6335 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6336 * out-of-sync. Make sure to update the required fields
6337 * before using them.
6338 *
6339 * @remarks No-long-jump zone!!!
6340 */
6341static int hmR0VmxSaveGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6342{
6343 int rc = VINF_SUCCESS;
6344 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP))
6345 {
6346 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RIP));
6347 uint64_t u64Val = 0;
6348 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6349 AssertRCReturn(rc, rc);
6350
6351 pMixedCtx->rip = u64Val;
6352 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP);
6353 }
6354 return rc;
6355}
6356
6357
6358/**
6359 * Saves the guest's RSP register from the VMCS into the guest-CPU context.
6360 *
6361 * @returns VBox status code.
6362 * @param pVCpu The cross context virtual CPU structure.
6363 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6364 * out-of-sync. Make sure to update the required fields
6365 * before using them.
6366 *
6367 * @remarks No-long-jump zone!!!
6368 */
6369static int hmR0VmxSaveGuestRsp(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6370{
6371 int rc = VINF_SUCCESS;
6372 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP))
6373 {
6374 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RSP));
6375 uint64_t u64Val = 0;
6376 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6377 AssertRCReturn(rc, rc);
6378
6379 pMixedCtx->rsp = u64Val;
6380 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RSP);
6381 }
6382 return rc;
6383}
6384
6385
6386/**
6387 * Saves the guest's RFLAGS from the VMCS into the guest-CPU context.
6388 *
6389 * @returns VBox status code.
6390 * @param pVCpu The cross context virtual CPU structure.
6391 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6392 * out-of-sync. Make sure to update the required fields
6393 * before using them.
6394 *
6395 * @remarks No-long-jump zone!!!
6396 */
6397static int hmR0VmxSaveGuestRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6398{
6399 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS))
6400 {
6401 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS));
6402 uint32_t uVal = 0;
6403 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &uVal);
6404 AssertRCReturn(rc, rc);
6405
6406 pMixedCtx->eflags.u32 = uVal;
6407 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active) /* Undo our real-on-v86-mode changes to eflags if necessary. */
6408 {
6409 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
6410 Log4(("Saving real-mode EFLAGS VT-x view=%#RX32\n", pMixedCtx->eflags.u32));
6411
6412 pMixedCtx->eflags.Bits.u1VM = 0;
6413 pMixedCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6414 }
6415
6416 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS);
6417 }
6418 return VINF_SUCCESS;
6419}
6420
6421
6422/**
6423 * Wrapper for saving the guest's RIP, RSP and RFLAGS from the VMCS into the
6424 * guest-CPU context.
6425 */
6426DECLINLINE(int) hmR0VmxSaveGuestRipRspRflags(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6427{
6428 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6429 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
6430 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
6431 return rc;
6432}
6433
6434
6435/**
6436 * Saves the guest's interruptibility-state ("interrupt shadow" as AMD calls it)
6437 * from the guest-state area in the VMCS.
6438 *
6439 * @param pVCpu The cross context virtual CPU structure.
6440 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6441 * out-of-sync. Make sure to update the required fields
6442 * before using them.
6443 *
6444 * @remarks No-long-jump zone!!!
6445 */
6446static void hmR0VmxSaveGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6447{
6448 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE))
6449 {
6450 uint32_t uIntrState = 0;
6451 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
6452 AssertRC(rc);
6453
6454 if (!uIntrState)
6455 {
6456 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6457 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6458
6459 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6460 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6461 }
6462 else
6463 {
6464 if (uIntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6465 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6466 {
6467 rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
6468 AssertRC(rc);
6469 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* for hmR0VmxGetGuestIntrState(). */
6470 AssertRC(rc);
6471
6472 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
6473 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
6474 }
6475 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6476 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6477
6478 if (uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6479 {
6480 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6481 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6482 }
6483 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6484 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6485 }
6486
6487 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_INTR_STATE);
6488 }
6489}
6490
6491
6492/**
6493 * Saves the guest's activity state.
6494 *
6495 * @returns VBox status code.
6496 * @param pVCpu The cross context virtual CPU structure.
6497 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6498 * out-of-sync. Make sure to update the required fields
6499 * before using them.
6500 *
6501 * @remarks No-long-jump zone!!!
6502 */
6503static int hmR0VmxSaveGuestActivityState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6504{
6505 NOREF(pMixedCtx);
6506 /* Nothing to do for now until we make use of different guest-CPU activity state. Just update the flag. */
6507 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_ACTIVITY_STATE);
6508 return VINF_SUCCESS;
6509}
6510
6511
6512/**
6513 * Saves the guest SYSENTER MSRs (SYSENTER_CS, SYSENTER_EIP, SYSENTER_ESP) from
6514 * the current VMCS into the guest-CPU context.
6515 *
6516 * @returns VBox status code.
6517 * @param pVCpu The cross context virtual CPU structure.
6518 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6519 * out-of-sync. Make sure to update the required fields
6520 * before using them.
6521 *
6522 * @remarks No-long-jump zone!!!
6523 */
6524static int hmR0VmxSaveGuestSysenterMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6525{
6526 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR))
6527 {
6528 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR));
6529 uint32_t u32Val = 0;
6530 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val); AssertRCReturn(rc, rc);
6531 pMixedCtx->SysEnter.cs = u32Val;
6532 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
6533 }
6534
6535 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR))
6536 {
6537 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR));
6538 uint64_t u64Val = 0;
6539 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &u64Val); AssertRCReturn(rc, rc);
6540 pMixedCtx->SysEnter.eip = u64Val;
6541 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
6542 }
6543 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR))
6544 {
6545 Assert(!HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR));
6546 uint64_t u64Val = 0;
6547 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &u64Val); AssertRCReturn(rc, rc);
6548 pMixedCtx->SysEnter.esp = u64Val;
6549 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
6550 }
6551 return VINF_SUCCESS;
6552}
6553
6554
6555/**
6556 * Saves the set of guest MSRs (that we restore lazily while leaving VT-x) from
6557 * the CPU back into the guest-CPU context.
6558 *
6559 * @returns VBox status code.
6560 * @param pVCpu The cross context virtual CPU structure.
6561 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6562 * out-of-sync. Make sure to update the required fields
6563 * before using them.
6564 *
6565 * @remarks No-long-jump zone!!!
6566 */
6567static int hmR0VmxSaveGuestLazyMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6568{
6569 /* Since this can be called from our preemption hook it's safer to make the guest-MSRs update non-preemptible. */
6570 VMMRZCallRing3Disable(pVCpu);
6571 HM_DISABLE_PREEMPT();
6572
6573 /* Doing the check here ensures we don't overwrite already-saved guest MSRs from a preemption hook. */
6574 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS))
6575 {
6576 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS));
6577 hmR0VmxLazySaveGuestMsrs(pVCpu, pMixedCtx);
6578 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS);
6579 }
6580
6581 HM_RESTORE_PREEMPT();
6582 VMMRZCallRing3Enable(pVCpu);
6583
6584 return VINF_SUCCESS;
6585}
6586
6587
6588/**
6589 * Saves the auto load/store'd guest MSRs from the current VMCS into
6590 * the guest-CPU context.
6591 *
6592 * @returns VBox status code.
6593 * @param pVCpu The cross context virtual CPU structure.
6594 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6595 * out-of-sync. Make sure to update the required fields
6596 * before using them.
6597 *
6598 * @remarks No-long-jump zone!!!
6599 */
6600static int hmR0VmxSaveGuestAutoLoadStoreMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6601{
6602 if (HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS))
6603 return VINF_SUCCESS;
6604
6605 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS));
6606 PVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6607 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
6608 Log4(("hmR0VmxSaveGuestAutoLoadStoreMsrs: cMsrs=%u\n", cMsrs));
6609 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6610 {
6611 switch (pMsr->u32Msr)
6612 {
6613 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6614 case MSR_K8_LSTAR: pMixedCtx->msrLSTAR = pMsr->u64Value; break;
6615 case MSR_K6_STAR: pMixedCtx->msrSTAR = pMsr->u64Value; break;
6616 case MSR_K8_SF_MASK: pMixedCtx->msrSFMASK = pMsr->u64Value; break;
6617 case MSR_K8_KERNEL_GS_BASE: pMixedCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6618 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6619 case MSR_K6_EFER: /* Nothing to do here since we intercept writes, see hmR0VmxLoadGuestMsrs(). */
6620 break;
6621
6622 default:
6623 {
6624 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr, cMsrs));
6625 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6626 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6627 }
6628 }
6629 }
6630
6631 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS);
6632 return VINF_SUCCESS;
6633}
6634
6635
6636/**
6637 * Saves the guest control registers from the current VMCS into the guest-CPU
6638 * context.
6639 *
6640 * @returns VBox status code.
6641 * @param pVCpu The cross context virtual CPU structure.
6642 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6643 * out-of-sync. Make sure to update the required fields
6644 * before using them.
6645 *
6646 * @remarks No-long-jump zone!!!
6647 */
6648static int hmR0VmxSaveGuestControlRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6649{
6650 /* Guest CR0. Guest FPU. */
6651 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6652 AssertRCReturn(rc, rc);
6653
6654 /* Guest CR4. */
6655 rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
6656 AssertRCReturn(rc, rc);
6657
6658 /* Guest CR2 - updated always during the world-switch or in #PF. */
6659 /* Guest CR3. Only changes with Nested Paging. This must be done -after- saving CR0 and CR4 from the guest! */
6660 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3))
6661 {
6662 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR3));
6663 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
6664 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR4));
6665
6666 PVM pVM = pVCpu->CTX_SUFF(pVM);
6667 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6668 || ( pVM->hm.s.fNestedPaging
6669 && CPUMIsGuestPagingEnabledEx(pMixedCtx)))
6670 {
6671 uint64_t u64Val = 0;
6672 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6673 if (pMixedCtx->cr3 != u64Val)
6674 {
6675 CPUMSetGuestCR3(pVCpu, u64Val);
6676 if (VMMRZCallRing3IsEnabled(pVCpu))
6677 {
6678 PGMUpdateCR3(pVCpu, u64Val);
6679 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6680 }
6681 else
6682 {
6683 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMUpdateCR3().*/
6684 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6685 }
6686 }
6687
6688 /* If the guest is in PAE mode, sync back the PDPE's into the guest state. */
6689 if (CPUMIsGuestInPAEModeEx(pMixedCtx)) /* Reads CR0, CR4 and EFER MSR (EFER is always up-to-date). */
6690 {
6691 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6692 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6693 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6694 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6695 AssertRCReturn(rc, rc);
6696
6697 if (VMMRZCallRing3IsEnabled(pVCpu))
6698 {
6699 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6700 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6701 }
6702 else
6703 {
6704 /* Set the force flag to inform PGM about it when necessary. It is cleared by PGMGstUpdatePaePdpes(). */
6705 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6706 }
6707 }
6708 }
6709
6710 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR3);
6711 }
6712
6713 /*
6714 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6715 * -> VMMRZCallRing3Disable() -> hmR0VmxSaveGuestState() -> Set VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6716 * -> continue with VM-exit handling -> hmR0VmxSaveGuestControlRegs() and here we are.
6717 *
6718 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6719 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6720 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6721 * -NOT- check if HMVMX_UPDATED_GUEST_CR3 is already set or not!
6722 *
6723 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6724 */
6725 if (VMMRZCallRing3IsEnabled(pVCpu))
6726 {
6727 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6728 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6729
6730 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6731 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6732
6733 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6734 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6735 }
6736
6737 return rc;
6738}
6739
6740
6741/**
6742 * Reads a guest segment register from the current VMCS into the guest-CPU
6743 * context.
6744 *
6745 * @returns VBox status code.
6746 * @param pVCpu The cross context virtual CPU structure.
6747 * @param idxSel Index of the selector in the VMCS.
6748 * @param idxLimit Index of the segment limit in the VMCS.
6749 * @param idxBase Index of the segment base in the VMCS.
6750 * @param idxAccess Index of the access rights of the segment in the VMCS.
6751 * @param pSelReg Pointer to the segment selector.
6752 *
6753 * @remarks No-long-jump zone!!!
6754 * @remarks Never call this function directly!!! Use the VMXLOCAL_READ_SEG()
6755 * macro as that takes care of whether to read from the VMCS cache or
6756 * not.
6757 */
6758DECLINLINE(int) hmR0VmxReadSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6759 PCPUMSELREG pSelReg)
6760{
6761 NOREF(pVCpu);
6762
6763 uint32_t u32Val = 0;
6764 int rc = VMXReadVmcs32(idxSel, &u32Val);
6765 AssertRCReturn(rc, rc);
6766 pSelReg->Sel = (uint16_t)u32Val;
6767 pSelReg->ValidSel = (uint16_t)u32Val;
6768 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6769
6770 rc = VMXReadVmcs32(idxLimit, &u32Val);
6771 AssertRCReturn(rc, rc);
6772 pSelReg->u32Limit = u32Val;
6773
6774 uint64_t u64Val = 0;
6775 rc = VMXReadVmcsGstNByIdxVal(idxBase, &u64Val);
6776 AssertRCReturn(rc, rc);
6777 pSelReg->u64Base = u64Val;
6778
6779 rc = VMXReadVmcs32(idxAccess, &u32Val);
6780 AssertRCReturn(rc, rc);
6781 pSelReg->Attr.u = u32Val;
6782
6783 /*
6784 * If VT-x marks the segment as unusable, most other bits remain undefined:
6785 * - For CS the L, D and G bits have meaning.
6786 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6787 * - For the remaining data segments no bits are defined.
6788 *
6789 * The present bit and the unusable bit has been observed to be set at the
6790 * same time (the selector was supposed to be invalid as we started executing
6791 * a V8086 interrupt in ring-0).
6792 *
6793 * What should be important for the rest of the VBox code, is that the P bit is
6794 * cleared. Some of the other VBox code recognizes the unusable bit, but
6795 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6796 * safe side here, we'll strip off P and other bits we don't care about. If
6797 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6798 *
6799 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6800 */
6801 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6802 {
6803 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6804
6805 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6806 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6807 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6808
6809 Log4(("hmR0VmxReadSegmentReg: Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Val, pSelReg->Attr.u));
6810#ifdef DEBUG_bird
6811 AssertMsg((u32Val & ~X86DESCATTR_P) == pSelReg->Attr.u,
6812 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6813 idxSel, u32Val, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6814#endif
6815 }
6816 return VINF_SUCCESS;
6817}
6818
6819
6820#ifdef VMX_USE_CACHED_VMCS_ACCESSES
6821# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6822 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6823 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6824#else
6825# define VMXLOCAL_READ_SEG(Sel, CtxSel) \
6826 hmR0VmxReadSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
6827 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, &pMixedCtx->CtxSel)
6828#endif
6829
6830
6831/**
6832 * Saves the guest segment registers from the current VMCS into the guest-CPU
6833 * context.
6834 *
6835 * @returns VBox status code.
6836 * @param pVCpu The cross context virtual CPU structure.
6837 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6838 * out-of-sync. Make sure to update the required fields
6839 * before using them.
6840 *
6841 * @remarks No-long-jump zone!!!
6842 */
6843static int hmR0VmxSaveGuestSegmentRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6844{
6845 /* Guest segment registers. */
6846 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS))
6847 {
6848 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS));
6849 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6850 AssertRCReturn(rc, rc);
6851
6852 rc = VMXLOCAL_READ_SEG(CS, cs);
6853 rc |= VMXLOCAL_READ_SEG(SS, ss);
6854 rc |= VMXLOCAL_READ_SEG(DS, ds);
6855 rc |= VMXLOCAL_READ_SEG(ES, es);
6856 rc |= VMXLOCAL_READ_SEG(FS, fs);
6857 rc |= VMXLOCAL_READ_SEG(GS, gs);
6858 AssertRCReturn(rc, rc);
6859
6860 /* Restore segment attributes for real-on-v86 mode hack. */
6861 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6862 {
6863 pMixedCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6864 pMixedCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6865 pMixedCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6866 pMixedCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6867 pMixedCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6868 pMixedCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6869 }
6870 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SEGMENT_REGS);
6871 }
6872
6873 return VINF_SUCCESS;
6874}
6875
6876
6877/**
6878 * Saves the guest descriptor table registers and task register from the current
6879 * VMCS into the guest-CPU context.
6880 *
6881 * @returns VBox status code.
6882 * @param pVCpu The cross context virtual CPU structure.
6883 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6884 * out-of-sync. Make sure to update the required fields
6885 * before using them.
6886 *
6887 * @remarks No-long-jump zone!!!
6888 */
6889static int hmR0VmxSaveGuestTableRegs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6890{
6891 int rc = VINF_SUCCESS;
6892
6893 /* Guest LDTR. */
6894 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR))
6895 {
6896 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_LDTR));
6897 rc = VMXLOCAL_READ_SEG(LDTR, ldtr);
6898 AssertRCReturn(rc, rc);
6899 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LDTR);
6900 }
6901
6902 /* Guest GDTR. */
6903 uint64_t u64Val = 0;
6904 uint32_t u32Val = 0;
6905 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR))
6906 {
6907 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_GDTR));
6908 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6909 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6910 pMixedCtx->gdtr.pGdt = u64Val;
6911 pMixedCtx->gdtr.cbGdt = u32Val;
6912 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_GDTR);
6913 }
6914
6915 /* Guest IDTR. */
6916 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR))
6917 {
6918 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_IDTR));
6919 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6920 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val); AssertRCReturn(rc, rc);
6921 pMixedCtx->idtr.pIdt = u64Val;
6922 pMixedCtx->idtr.cbIdt = u32Val;
6923 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_IDTR);
6924 }
6925
6926 /* Guest TR. */
6927 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR))
6928 {
6929 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_TR));
6930 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
6931 AssertRCReturn(rc, rc);
6932
6933 /* For real-mode emulation using virtual-8086 mode we have the fake TSS (pRealModeTSS) in TR, don't save the fake one. */
6934 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6935 {
6936 rc = VMXLOCAL_READ_SEG(TR, tr);
6937 AssertRCReturn(rc, rc);
6938 }
6939 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_TR);
6940 }
6941 return rc;
6942}
6943
6944#undef VMXLOCAL_READ_SEG
6945
6946
6947/**
6948 * Saves the guest debug-register DR7 from the current VMCS into the guest-CPU
6949 * context.
6950 *
6951 * @returns VBox status code.
6952 * @param pVCpu The cross context virtual CPU structure.
6953 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6954 * out-of-sync. Make sure to update the required fields
6955 * before using them.
6956 *
6957 * @remarks No-long-jump zone!!!
6958 */
6959static int hmR0VmxSaveGuestDR7(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6960{
6961 if (!HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7))
6962 {
6963 if (!pVCpu->hm.s.fUsingHyperDR7)
6964 {
6965 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6966 uint32_t u32Val;
6967 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val); AssertRCReturn(rc, rc);
6968 pMixedCtx->dr[7] = u32Val;
6969 }
6970
6971 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_DR7);
6972 }
6973 return VINF_SUCCESS;
6974}
6975
6976
6977/**
6978 * Saves the guest APIC state from the current VMCS into the guest-CPU context.
6979 *
6980 * @returns VBox status code.
6981 * @param pVCpu The cross context virtual CPU structure.
6982 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
6983 * out-of-sync. Make sure to update the required fields
6984 * before using them.
6985 *
6986 * @remarks No-long-jump zone!!!
6987 */
6988static int hmR0VmxSaveGuestApicState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6989{
6990 NOREF(pMixedCtx);
6991
6992 /* Updating TPR is already done in hmR0VmxPostRunGuest(). Just update the flag. */
6993 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_APIC_STATE);
6994 return VINF_SUCCESS;
6995}
6996
6997
6998/**
6999 * Saves the entire guest state from the currently active VMCS into the
7000 * guest-CPU context.
7001 *
7002 * This essentially VMREADs all guest-data.
7003 *
7004 * @returns VBox status code.
7005 * @param pVCpu The cross context virtual CPU structure.
7006 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7007 * out-of-sync. Make sure to update the required fields
7008 * before using them.
7009 */
7010static int hmR0VmxSaveGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7011{
7012 Assert(pVCpu);
7013 Assert(pMixedCtx);
7014
7015 if (HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL)
7016 return VINF_SUCCESS;
7017
7018 /* Though we can longjmp to ring-3 due to log-flushes here and get recalled
7019 again on the ring-3 callback path, there is no real need to. */
7020 if (VMMRZCallRing3IsEnabled(pVCpu))
7021 VMMR0LogFlushDisable(pVCpu);
7022 else
7023 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7024 Log4Func(("vcpu[%RU32]\n", pVCpu->idCpu));
7025
7026 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
7027 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestRipRspRflags failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7028
7029 rc = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7030 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestControlRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7031
7032 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7033 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSegmentRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7034
7035 rc = hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
7036 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestTableRegs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7037
7038 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
7039 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestDR7 failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7040
7041 rc = hmR0VmxSaveGuestSysenterMsrs(pVCpu, pMixedCtx);
7042 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestSysenterMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7043
7044 rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7045 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestLazyMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7046
7047 rc = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
7048 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestAutoLoadStoreMsrs failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7049
7050 rc = hmR0VmxSaveGuestActivityState(pVCpu, pMixedCtx);
7051 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestActivityState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7052
7053 rc = hmR0VmxSaveGuestApicState(pVCpu, pMixedCtx);
7054 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveGuestApicState failed! rc=%Rrc (pVCpu=%p)\n", rc, pVCpu), rc);
7055
7056 AssertMsg(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL,
7057 ("Missed guest state bits while saving state; missing %RX32 (got %RX32, want %RX32) - check log for any previous errors!\n",
7058 HMVMX_UPDATED_GUEST_ALL ^ HMVMXCPU_GST_VALUE(pVCpu), HMVMXCPU_GST_VALUE(pVCpu), HMVMX_UPDATED_GUEST_ALL));
7059
7060 if (VMMRZCallRing3IsEnabled(pVCpu))
7061 VMMR0LogFlushEnable(pVCpu);
7062
7063 return VINF_SUCCESS;
7064}
7065
7066
7067/**
7068 * Saves basic guest registers needed for IEM instruction execution.
7069 *
7070 * @returns VBox status code (OR-able).
7071 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7072 * @param pMixedCtx Pointer to the CPU context of the guest.
7073 * @param fMemory Whether the instruction being executed operates on
7074 * memory or not. Only CR0 is synced up if clear.
7075 * @param fNeedRsp Need RSP (any instruction working on GPRs or stack).
7076 */
7077static int hmR0VmxSaveGuestRegsForIemExec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fMemory, bool fNeedRsp)
7078{
7079 /*
7080 * We assume all general purpose registers other than RSP are available.
7081 *
7082 * - RIP is a must, as it will be incremented or otherwise changed.
7083 * - RFLAGS are always required to figure the CPL.
7084 * - RSP isn't always required, however it's a GPR, so frequently required.
7085 * - SS and CS are the only segment register needed if IEM doesn't do memory
7086 * access (CPL + 16/32/64-bit mode), but we can only get all segment registers.
7087 * - CR0 is always required by IEM for the CPL, while CR3 and CR4 will only
7088 * be required for memory accesses.
7089 *
7090 * Note! Before IEM dispatches an exception, it will call us to sync in everything.
7091 */
7092 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
7093 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7094 if (fNeedRsp)
7095 rc |= hmR0VmxSaveGuestRsp(pVCpu, pMixedCtx);
7096 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
7097 if (!fMemory)
7098 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7099 else
7100 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7101 AssertRCReturn(rc, rc);
7102 return rc;
7103}
7104
7105
7106/**
7107 * Ensures that we've got a complete basic guest-context.
7108 *
7109 * This excludes the FPU, SSE, AVX, and similar extended state. The interface
7110 * is for the interpreter.
7111 *
7112 * @returns VBox status code.
7113 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
7114 * @param pMixedCtx Pointer to the guest-CPU context which may have data
7115 * needing to be synced in.
7116 * @thread EMT(pVCpu)
7117 */
7118VMMR0_INT_DECL(int) HMR0EnsureCompleteBasicContext(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7119{
7120 /* Note! Since this is only applicable to VT-x, the implementation is placed
7121 in the VT-x part of the sources instead of the generic stuff. */
7122 int rc;
7123 PVM pVM = pVCpu->CTX_SUFF(pVM);
7124 if ( pVM->hm.s.vmx.fSupported
7125 && VM_IS_HM_ENABLED(pVM))
7126 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7127 else
7128 rc = VINF_SUCCESS;
7129
7130 /*
7131 * For now, imply that the caller might change everything too. Do this after
7132 * saving the guest state so as to not trigger assertions.
7133 *
7134 * This is required for AMD-V too as it too only selectively re-loads changed
7135 * guest state back in to the VMCB.
7136 */
7137 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7138 return rc;
7139}
7140
7141
7142/**
7143 * Check per-VM and per-VCPU force flag actions that require us to go back to
7144 * ring-3 for one reason or another.
7145 *
7146 * @returns Strict VBox status code (i.e. informational status codes too)
7147 * @retval VINF_SUCCESS if we don't have any actions that require going back to
7148 * ring-3.
7149 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
7150 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
7151 * interrupts)
7152 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
7153 * all EMTs to be in ring-3.
7154 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
7155 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
7156 * to the EM loop.
7157 *
7158 * @param pVM The cross context VM structure.
7159 * @param pVCpu The cross context virtual CPU structure.
7160 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7161 * out-of-sync. Make sure to update the required fields
7162 * before using them.
7163 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
7164 */
7165static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
7166{
7167 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7168
7169 /*
7170 * Anything pending? Should be more likely than not if we're doing a good job.
7171 */
7172 if ( !fStepping
7173 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
7174 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
7175 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
7176 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
7177 return VINF_SUCCESS;
7178
7179 /* We need the control registers now, make sure the guest-CPU context is updated. */
7180 int rc3 = hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
7181 AssertRCReturn(rc3, rc3);
7182
7183 /* Pending HM CR3 sync. */
7184 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
7185 {
7186 int rc2 = PGMUpdateCR3(pVCpu, pMixedCtx->cr3);
7187 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
7188 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
7189 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
7190 }
7191
7192 /* Pending HM PAE PDPEs. */
7193 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
7194 {
7195 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
7196 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
7197 }
7198
7199 /* Pending PGM C3 sync. */
7200 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
7201 {
7202 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
7203 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
7204 if (rcStrict2 != VINF_SUCCESS)
7205 {
7206 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
7207 Log4(("hmR0VmxCheckForceFlags: PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
7208 return rcStrict2;
7209 }
7210 }
7211
7212 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
7213 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
7214 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
7215 {
7216 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
7217 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
7218 Log4(("hmR0VmxCheckForceFlags: HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
7219 return rc2;
7220 }
7221
7222 /* Pending VM request packets, such as hardware interrupts. */
7223 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
7224 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
7225 {
7226 Log4(("hmR0VmxCheckForceFlags: Pending VM request forcing us back to ring-3\n"));
7227 return VINF_EM_PENDING_REQUEST;
7228 }
7229
7230 /* Pending PGM pool flushes. */
7231 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
7232 {
7233 Log4(("hmR0VmxCheckForceFlags: PGM pool flush pending forcing us back to ring-3\n"));
7234 return VINF_PGM_POOL_FLUSH_PENDING;
7235 }
7236
7237 /* Pending DMA requests. */
7238 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
7239 {
7240 Log4(("hmR0VmxCheckForceFlags: Pending DMA request forcing us back to ring-3\n"));
7241 return VINF_EM_RAW_TO_R3;
7242 }
7243
7244 return VINF_SUCCESS;
7245}
7246
7247
7248/**
7249 * Converts any TRPM trap into a pending HM event. This is typically used when
7250 * entering from ring-3 (not longjmp returns).
7251 *
7252 * @param pVCpu The cross context virtual CPU structure.
7253 */
7254static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
7255{
7256 Assert(TRPMHasTrap(pVCpu));
7257 Assert(!pVCpu->hm.s.Event.fPending);
7258
7259 uint8_t uVector;
7260 TRPMEVENT enmTrpmEvent;
7261 RTGCUINT uErrCode;
7262 RTGCUINTPTR GCPtrFaultAddress;
7263 uint8_t cbInstr;
7264
7265 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
7266 AssertRC(rc);
7267
7268 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
7269 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7270 if (enmTrpmEvent == TRPM_TRAP)
7271 {
7272 switch (uVector)
7273 {
7274 case X86_XCPT_NMI:
7275 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7276 break;
7277
7278 case X86_XCPT_BP:
7279 case X86_XCPT_OF:
7280 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7281 break;
7282
7283 case X86_XCPT_PF:
7284 case X86_XCPT_DF:
7285 case X86_XCPT_TS:
7286 case X86_XCPT_NP:
7287 case X86_XCPT_SS:
7288 case X86_XCPT_GP:
7289 case X86_XCPT_AC:
7290 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7291 RT_FALL_THRU();
7292 default:
7293 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7294 break;
7295 }
7296 }
7297 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
7298 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7299 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
7300 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7301 else
7302 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
7303
7304 rc = TRPMResetTrap(pVCpu);
7305 AssertRC(rc);
7306 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
7307 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
7308
7309 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
7310}
7311
7312
7313/**
7314 * Converts the pending HM event into a TRPM trap.
7315 *
7316 * @param pVCpu The cross context virtual CPU structure.
7317 */
7318static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
7319{
7320 Assert(pVCpu->hm.s.Event.fPending);
7321
7322 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7323 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
7324 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
7325 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
7326
7327 /* If a trap was already pending, we did something wrong! */
7328 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
7329
7330 TRPMEVENT enmTrapType;
7331 switch (uVectorType)
7332 {
7333 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
7334 enmTrapType = TRPM_HARDWARE_INT;
7335 break;
7336
7337 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
7338 enmTrapType = TRPM_SOFTWARE_INT;
7339 break;
7340
7341 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
7342 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
7343 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
7344 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
7345 enmTrapType = TRPM_TRAP;
7346 break;
7347
7348 default:
7349 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
7350 enmTrapType = TRPM_32BIT_HACK;
7351 break;
7352 }
7353
7354 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
7355
7356 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
7357 AssertRC(rc);
7358
7359 if (fErrorCodeValid)
7360 TRPMSetErrorCode(pVCpu, uErrorCode);
7361
7362 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
7363 && uVector == X86_XCPT_PF)
7364 {
7365 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
7366 }
7367 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7368 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
7369 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
7370 {
7371 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
7372 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
7373 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
7374 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
7375 }
7376
7377 /* Clear any pending events from the VMCS. */
7378 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
7379 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
7380
7381 /* We're now done converting the pending event. */
7382 pVCpu->hm.s.Event.fPending = false;
7383}
7384
7385
7386/**
7387 * Does the necessary state syncing before returning to ring-3 for any reason
7388 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
7389 *
7390 * @returns VBox status code.
7391 * @param pVCpu The cross context virtual CPU structure.
7392 * @param pMixedCtx Pointer to the guest-CPU context. The data may
7393 * be out-of-sync. Make sure to update the required
7394 * fields before using them.
7395 * @param fSaveGuestState Whether to save the guest state or not.
7396 *
7397 * @remarks No-long-jmp zone!!!
7398 */
7399static int hmR0VmxLeave(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fSaveGuestState)
7400{
7401 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7402 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7403
7404 RTCPUID idCpu = RTMpCpuId();
7405 Log4Func(("HostCpuId=%u\n", idCpu));
7406
7407 /*
7408 * !!! IMPORTANT !!!
7409 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
7410 */
7411
7412 /* Save the guest state if necessary. */
7413 if ( fSaveGuestState
7414 && HMVMXCPU_GST_VALUE(pVCpu) != HMVMX_UPDATED_GUEST_ALL)
7415 {
7416 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
7417 AssertRCReturn(rc, rc);
7418 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7419 }
7420
7421 /* Restore host FPU state if necessary and resync on next R0 reentry .*/
7422 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu))
7423 {
7424 /* We shouldn't reload CR0 without saving it first. */
7425 if (!fSaveGuestState)
7426 {
7427 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
7428 AssertRCReturn(rc, rc);
7429 }
7430 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
7431 }
7432
7433 /* Restore host debug registers if necessary and resync on next R0 reentry. */
7434#ifdef VBOX_STRICT
7435 if (CPUMIsHyperDebugStateActive(pVCpu))
7436 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
7437#endif
7438 if (CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */))
7439 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
7440 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
7441 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
7442
7443#if HC_ARCH_BITS == 64
7444 /* Restore host-state bits that VT-x only restores partially. */
7445 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7446 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7447 {
7448 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
7449 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7450 }
7451 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7452#endif
7453
7454 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7455 if (pVCpu->hm.s.vmx.fLazyMsrs)
7456 {
7457 /* We shouldn't reload the guest MSRs without saving it first. */
7458 if (!fSaveGuestState)
7459 {
7460 int rc = hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
7461 AssertRCReturn(rc, rc);
7462 }
7463 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_LAZY_MSRS));
7464 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7465 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
7466 }
7467
7468 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7469 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7470
7471 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
7472 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatLoadGuestState);
7473 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit1);
7474 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExit2);
7475 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
7476 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
7477 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
7478 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7479
7480 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7481
7482 /** @todo This partially defeats the purpose of having preemption hooks.
7483 * The problem is, deregistering the hooks should be moved to a place that
7484 * lasts until the EMT is about to be destroyed not everytime while leaving HM
7485 * context.
7486 */
7487 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7488 {
7489 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7490 AssertRCReturn(rc, rc);
7491
7492 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7493 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
7494 }
7495 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
7496 NOREF(idCpu);
7497
7498 return VINF_SUCCESS;
7499}
7500
7501
7502/**
7503 * Leaves the VT-x session.
7504 *
7505 * @returns VBox status code.
7506 * @param pVCpu The cross context virtual CPU structure.
7507 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7508 * out-of-sync. Make sure to update the required fields
7509 * before using them.
7510 *
7511 * @remarks No-long-jmp zone!!!
7512 */
7513DECLINLINE(int) hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7514{
7515 HM_DISABLE_PREEMPT();
7516 HMVMX_ASSERT_CPU_SAFE();
7517 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
7518 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7519
7520 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
7521 and done this from the VMXR0ThreadCtxCallback(). */
7522 if (!pVCpu->hm.s.fLeaveDone)
7523 {
7524 int rc2 = hmR0VmxLeave(pVCpu, pMixedCtx, true /* fSaveGuestState */);
7525 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
7526 pVCpu->hm.s.fLeaveDone = true;
7527 }
7528 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
7529
7530 /*
7531 * !!! IMPORTANT !!!
7532 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
7533 */
7534
7535 /* Deregister hook now that we've left HM context before re-enabling preemption. */
7536 /** @todo Deregistering here means we need to VMCLEAR always
7537 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
7538 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7539 VMMR0ThreadCtxHookDisable(pVCpu);
7540
7541 /* Leave HM context. This takes care of local init (term). */
7542 int rc = HMR0LeaveCpu(pVCpu);
7543
7544 HM_RESTORE_PREEMPT();
7545 return rc;
7546}
7547
7548
7549/**
7550 * Does the necessary state syncing before doing a longjmp to ring-3.
7551 *
7552 * @returns VBox status code.
7553 * @param pVCpu The cross context virtual CPU structure.
7554 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7555 * out-of-sync. Make sure to update the required fields
7556 * before using them.
7557 *
7558 * @remarks No-long-jmp zone!!!
7559 */
7560DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7561{
7562 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7563}
7564
7565
7566/**
7567 * Take necessary actions before going back to ring-3.
7568 *
7569 * An action requires us to go back to ring-3. This function does the necessary
7570 * steps before we can safely return to ring-3. This is not the same as longjmps
7571 * to ring-3, this is voluntary and prepares the guest so it may continue
7572 * executing outside HM (recompiler/IEM).
7573 *
7574 * @returns VBox status code.
7575 * @param pVM The cross context VM structure.
7576 * @param pVCpu The cross context virtual CPU structure.
7577 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7578 * out-of-sync. Make sure to update the required fields
7579 * before using them.
7580 * @param rcExit The reason for exiting to ring-3. Can be
7581 * VINF_VMM_UNKNOWN_RING3_CALL.
7582 */
7583static int hmR0VmxExitToRing3(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7584{
7585 Assert(pVM);
7586 Assert(pVCpu);
7587 Assert(pMixedCtx);
7588 HMVMX_ASSERT_PREEMPT_SAFE();
7589
7590 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7591 {
7592 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7593 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7594 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7595 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7596 }
7597
7598 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7599 VMMRZCallRing3Disable(pVCpu);
7600 Log4(("hmR0VmxExitToRing3: pVCpu=%p idCpu=%RU32 rcExit=%d\n", pVCpu, pVCpu->idCpu, VBOXSTRICTRC_VAL(rcExit)));
7601
7602 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7603 if (pVCpu->hm.s.Event.fPending)
7604 {
7605 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7606 Assert(!pVCpu->hm.s.Event.fPending);
7607 }
7608
7609 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7610 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7611
7612 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7613 and if we're injecting an event we should have a TRPM trap pending. */
7614 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7615#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a tripple fault in progress. */
7616 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7617#endif
7618
7619 /* Save guest state and restore host state bits. */
7620 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7621 AssertRCReturn(rc, rc);
7622 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7623 /* Thread-context hooks are unregistered at this point!!! */
7624
7625 /* Sync recompiler state. */
7626 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7627 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7628 | CPUM_CHANGED_LDTR
7629 | CPUM_CHANGED_GDTR
7630 | CPUM_CHANGED_IDTR
7631 | CPUM_CHANGED_TR
7632 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7633 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
7634 if ( pVM->hm.s.fNestedPaging
7635 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7636 {
7637 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7638 }
7639
7640 Assert(!pVCpu->hm.s.fClearTrapFlag);
7641
7642 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7643 if (rcExit != VINF_EM_RAW_INTERRUPT)
7644 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
7645
7646 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7647
7648 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7649 VMMRZCallRing3RemoveNotification(pVCpu);
7650 VMMRZCallRing3Enable(pVCpu);
7651
7652 return rc;
7653}
7654
7655
7656/**
7657 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7658 * longjump to ring-3 and possibly get preempted.
7659 *
7660 * @returns VBox status code.
7661 * @param pVCpu The cross context virtual CPU structure.
7662 * @param enmOperation The operation causing the ring-3 longjump.
7663 * @param pvUser Opaque pointer to the guest-CPU context. The data
7664 * may be out-of-sync. Make sure to update the required
7665 * fields before using them.
7666 */
7667static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7668{
7669 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7670 {
7671 /*
7672 * !!! IMPORTANT !!!
7673 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7674 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7675 */
7676 VMMRZCallRing3RemoveNotification(pVCpu);
7677 VMMRZCallRing3Disable(pVCpu);
7678 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7679 RTThreadPreemptDisable(&PreemptState);
7680
7681 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7682 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7683
7684#if HC_ARCH_BITS == 64
7685 /* Restore host-state bits that VT-x only restores partially. */
7686 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7687 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7688 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7689 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7690#endif
7691 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7692 if (pVCpu->hm.s.vmx.fLazyMsrs)
7693 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7694
7695 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7696 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7697 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7698 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7699 {
7700 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7701 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7702 }
7703
7704 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7705 VMMR0ThreadCtxHookDisable(pVCpu);
7706 HMR0LeaveCpu(pVCpu);
7707 RTThreadPreemptRestore(&PreemptState);
7708 return VINF_SUCCESS;
7709 }
7710
7711 Assert(pVCpu);
7712 Assert(pvUser);
7713 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7714 HMVMX_ASSERT_PREEMPT_SAFE();
7715
7716 VMMRZCallRing3Disable(pVCpu);
7717 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7718
7719 Log4(("hmR0VmxCallRing3Callback->hmR0VmxLongJmpToRing3 pVCpu=%p idCpu=%RU32 enmOperation=%d\n", pVCpu, pVCpu->idCpu,
7720 enmOperation));
7721
7722 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7723 AssertRCReturn(rc, rc);
7724
7725 VMMRZCallRing3Enable(pVCpu);
7726 return VINF_SUCCESS;
7727}
7728
7729
7730/**
7731 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7732 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7733 *
7734 * @param pVCpu The cross context virtual CPU structure.
7735 */
7736DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7737{
7738 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7739 {
7740 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7741 {
7742 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7743 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7744 AssertRC(rc);
7745 Log4(("Setup interrupt-window exiting\n"));
7746 }
7747 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7748}
7749
7750
7751/**
7752 * Clears the interrupt-window exiting control in the VMCS.
7753 *
7754 * @param pVCpu The cross context virtual CPU structure.
7755 */
7756DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7757{
7758 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7759 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7760 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7761 AssertRC(rc);
7762 Log4(("Cleared interrupt-window exiting\n"));
7763}
7764
7765
7766/**
7767 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7768 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7769 *
7770 * @param pVCpu The cross context virtual CPU structure.
7771 */
7772DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7773{
7774 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7775 {
7776 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7777 {
7778 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7779 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7780 AssertRC(rc);
7781 Log4(("Setup NMI-window exiting\n"));
7782 }
7783 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7784}
7785
7786
7787/**
7788 * Clears the NMI-window exiting control in the VMCS.
7789 *
7790 * @param pVCpu The cross context virtual CPU structure.
7791 */
7792DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7793{
7794 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7795 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7796 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7797 AssertRC(rc);
7798 Log4(("Cleared NMI-window exiting\n"));
7799}
7800
7801
7802/**
7803 * Evaluates the event to be delivered to the guest and sets it as the pending
7804 * event.
7805 *
7806 * @returns The VT-x guest-interruptibility state.
7807 * @param pVCpu The cross context virtual CPU structure.
7808 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7809 * out-of-sync. Make sure to update the required fields
7810 * before using them.
7811 */
7812static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7813{
7814 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7815 uint32_t const uIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7816 bool const fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7817 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7818 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7819
7820 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7821 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7822 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7823 Assert(!TRPMHasTrap(pVCpu));
7824
7825 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7826 APICUpdatePendingInterrupts(pVCpu);
7827
7828 /*
7829 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7830 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7831 */
7832 /** @todo SMI. SMIs take priority over NMIs. */
7833 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7834 {
7835 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7836 if ( !pVCpu->hm.s.Event.fPending
7837 && !fBlockNmi
7838 && !fBlockSti
7839 && !fBlockMovSS)
7840 {
7841 Log4(("Pending NMI vcpu[%RU32]\n", pVCpu->idCpu));
7842 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7843 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7844
7845 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7846 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7847 }
7848 else
7849 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7850 }
7851 /*
7852 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7853 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7854 */
7855 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7856 && !pVCpu->hm.s.fSingleInstruction)
7857 {
7858 Assert(!DBGFIsStepping(pVCpu));
7859 int rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7860 AssertRC(rc);
7861 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7862 if ( !pVCpu->hm.s.Event.fPending
7863 && !fBlockInt
7864 && !fBlockSti
7865 && !fBlockMovSS)
7866 {
7867 uint8_t u8Interrupt;
7868 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7869 if (RT_SUCCESS(rc))
7870 {
7871 Log4(("Pending interrupt vcpu[%RU32] u8Interrupt=%#x \n", pVCpu->idCpu, u8Interrupt));
7872 uint32_t u32IntInfo = u8Interrupt | VMX_EXIT_INTERRUPTION_INFO_VALID;
7873 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7874
7875 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7876 }
7877 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7878 {
7879 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7880 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7881 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7882
7883 /*
7884 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7885 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7886 * need to re-set this force-flag here.
7887 */
7888 }
7889 else
7890 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7891 }
7892 else
7893 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7894 }
7895
7896 return uIntrState;
7897}
7898
7899
7900/**
7901 * Sets a pending-debug exception to be delivered to the guest if the guest is
7902 * single-stepping in the VMCS.
7903 *
7904 * @param pVCpu The cross context virtual CPU structure.
7905 */
7906DECLINLINE(void) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7907{
7908 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS)); NOREF(pVCpu);
7909 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7910 AssertRC(rc);
7911}
7912
7913
7914/**
7915 * Injects any pending events into the guest if the guest is in a state to
7916 * receive them.
7917 *
7918 * @returns Strict VBox status code (i.e. informational status codes too).
7919 * @param pVCpu The cross context virtual CPU structure.
7920 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7921 * out-of-sync. Make sure to update the required fields
7922 * before using them.
7923 * @param uIntrState The VT-x guest-interruptibility state.
7924 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7925 * return VINF_EM_DBG_STEPPED if the event was
7926 * dispatched directly.
7927 */
7928static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t uIntrState, bool fStepping)
7929{
7930 HMVMX_ASSERT_PREEMPT_SAFE();
7931 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7932
7933 bool fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7934 bool fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7935
7936 Assert(!fBlockSti || HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RFLAGS));
7937 Assert(!(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7938 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7939 Assert(!TRPMHasTrap(pVCpu));
7940
7941 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7942 if (pVCpu->hm.s.Event.fPending)
7943 {
7944 /*
7945 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7946 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7947 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7948 *
7949 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7950 */
7951 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7952#ifdef VBOX_STRICT
7953 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7954 {
7955 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7956 Assert(!fBlockInt);
7957 Assert(!fBlockSti);
7958 Assert(!fBlockMovSS);
7959 }
7960 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7961 {
7962 bool const fBlockNmi = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7963 Assert(!fBlockSti);
7964 Assert(!fBlockMovSS);
7965 Assert(!fBlockNmi);
7966 }
7967#endif
7968 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#x\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7969 (uint8_t)uIntType));
7970 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7971 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress,
7972 fStepping, &uIntrState);
7973 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7974
7975 /* Update the interruptibility-state as it could have been changed by
7976 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7977 fBlockMovSS = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7978 fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7979
7980 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7981 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7982 else
7983 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7984 }
7985
7986 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7987 if ( fBlockSti
7988 || fBlockMovSS)
7989 {
7990 if (!pVCpu->hm.s.fSingleInstruction)
7991 {
7992 /*
7993 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7994 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7995 * See Intel spec. 27.3.4 "Saving Non-Register State".
7996 */
7997 Assert(!DBGFIsStepping(pVCpu));
7998 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
7999 AssertRCReturn(rc2, rc2);
8000 if (pMixedCtx->eflags.Bits.u1TF)
8001 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
8002 }
8003 else if (pMixedCtx->eflags.Bits.u1TF)
8004 {
8005 /*
8006 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
8007 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
8008 */
8009 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
8010 uIntrState = 0;
8011 }
8012 }
8013
8014 /*
8015 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
8016 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
8017 */
8018 int rc2 = hmR0VmxLoadGuestIntrState(pVCpu, uIntrState);
8019 AssertRC(rc2);
8020
8021 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
8022 NOREF(fBlockMovSS); NOREF(fBlockSti);
8023 return rcStrict;
8024}
8025
8026
8027/**
8028 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
8029 *
8030 * @param pVCpu The cross context virtual CPU structure.
8031 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8032 * out-of-sync. Make sure to update the required fields
8033 * before using them.
8034 */
8035DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8036{
8037 NOREF(pMixedCtx);
8038 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
8039 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8040}
8041
8042
8043/**
8044 * Injects a double-fault (\#DF) exception into the VM.
8045 *
8046 * @returns Strict VBox status code (i.e. informational status codes too).
8047 * @param pVCpu The cross context virtual CPU structure.
8048 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8049 * out-of-sync. Make sure to update the required fields
8050 * before using them.
8051 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
8052 * and should return VINF_EM_DBG_STEPPED if the event
8053 * is injected directly (register modified by us, not
8054 * by hardware on VM-entry).
8055 * @param puIntrState Pointer to the current guest interruptibility-state.
8056 * This interruptibility-state will be updated if
8057 * necessary. This cannot not be NULL.
8058 */
8059DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping, uint32_t *puIntrState)
8060{
8061 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8062 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8063 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8064 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */,
8065 fStepping, puIntrState);
8066}
8067
8068
8069/**
8070 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
8071 *
8072 * @param pVCpu The cross context virtual CPU structure.
8073 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8074 * out-of-sync. Make sure to update the required fields
8075 * before using them.
8076 */
8077DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8078{
8079 NOREF(pMixedCtx);
8080 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
8081 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8082 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8083}
8084
8085
8086/**
8087 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
8088 *
8089 * @param pVCpu The cross context virtual CPU structure.
8090 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8091 * out-of-sync. Make sure to update the required fields
8092 * before using them.
8093 * @param cbInstr The value of RIP that is to be pushed on the guest
8094 * stack.
8095 */
8096DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
8097{
8098 NOREF(pMixedCtx);
8099 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
8100 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8101 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8102}
8103
8104
8105/**
8106 * Injects a general-protection (\#GP) fault into the VM.
8107 *
8108 * @returns Strict VBox status code (i.e. informational status codes too).
8109 * @param pVCpu The cross context virtual CPU structure.
8110 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8111 * out-of-sync. Make sure to update the required fields
8112 * before using them.
8113 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
8114 * mode, i.e. in real-mode it's not valid).
8115 * @param u32ErrorCode The error code associated with the \#GP.
8116 * @param fStepping Whether we're running in
8117 * hmR0VmxRunGuestCodeStep() and should return
8118 * VINF_EM_DBG_STEPPED if the event is injected
8119 * directly (register modified by us, not by
8120 * hardware on VM-entry).
8121 * @param puIntrState Pointer to the current guest interruptibility-state.
8122 * This interruptibility-state will be updated if
8123 * necessary. This cannot not be NULL.
8124 */
8125DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
8126 bool fStepping, uint32_t *puIntrState)
8127{
8128 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8129 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8130 if (fErrorCodeValid)
8131 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8132 return hmR0VmxInjectEventVmcs(pVCpu, pMixedCtx, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */,
8133 fStepping, puIntrState);
8134}
8135
8136
8137#if 0 /* unused */
8138/**
8139 * Sets a general-protection (\#GP) exception as pending-for-injection into the
8140 * VM.
8141 *
8142 * @param pVCpu The cross context virtual CPU structure.
8143 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8144 * out-of-sync. Make sure to update the required fields
8145 * before using them.
8146 * @param u32ErrorCode The error code associated with the \#GP.
8147 */
8148DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
8149{
8150 NOREF(pMixedCtx);
8151 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
8152 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8153 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8154 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
8155}
8156#endif /* unused */
8157
8158
8159/**
8160 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
8161 *
8162 * @param pVCpu The cross context virtual CPU structure.
8163 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8164 * out-of-sync. Make sure to update the required fields
8165 * before using them.
8166 * @param uVector The software interrupt vector number.
8167 * @param cbInstr The value of RIP that is to be pushed on the guest
8168 * stack.
8169 */
8170DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
8171{
8172 NOREF(pMixedCtx);
8173 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
8174 if ( uVector == X86_XCPT_BP
8175 || uVector == X86_XCPT_OF)
8176 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8177 else
8178 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
8179 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
8180}
8181
8182
8183/**
8184 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
8185 * stack.
8186 *
8187 * @returns Strict VBox status code (i.e. informational status codes too).
8188 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
8189 * @param pVM The cross context VM structure.
8190 * @param pMixedCtx Pointer to the guest-CPU context.
8191 * @param uValue The value to push to the guest stack.
8192 */
8193DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
8194{
8195 /*
8196 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
8197 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
8198 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
8199 */
8200 if (pMixedCtx->sp == 1)
8201 return VINF_EM_RESET;
8202 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
8203 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
8204 AssertRC(rc);
8205 return rc;
8206}
8207
8208
8209/**
8210 * Injects an event into the guest upon VM-entry by updating the relevant fields
8211 * in the VM-entry area in the VMCS.
8212 *
8213 * @returns Strict VBox status code (i.e. informational status codes too).
8214 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
8215 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
8216 *
8217 * @param pVCpu The cross context virtual CPU structure.
8218 * @param pMixedCtx Pointer to the guest-CPU context. The data may
8219 * be out-of-sync. Make sure to update the required
8220 * fields before using them.
8221 * @param u64IntInfo The VM-entry interruption-information field.
8222 * @param cbInstr The VM-entry instruction length in bytes (for
8223 * software interrupts, exceptions and privileged
8224 * software exceptions).
8225 * @param u32ErrCode The VM-entry exception error code.
8226 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
8227 * @param puIntrState Pointer to the current guest interruptibility-state.
8228 * This interruptibility-state will be updated if
8229 * necessary. This cannot not be NULL.
8230 * @param fStepping Whether we're running in
8231 * hmR0VmxRunGuestCodeStep() and should return
8232 * VINF_EM_DBG_STEPPED if the event is injected
8233 * directly (register modified by us, not by
8234 * hardware on VM-entry).
8235 *
8236 * @remarks Requires CR0!
8237 */
8238static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint64_t u64IntInfo, uint32_t cbInstr,
8239 uint32_t u32ErrCode, RTGCUINTREG GCPtrFaultAddress, bool fStepping,
8240 uint32_t *puIntrState)
8241{
8242 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
8243 AssertMsg(u64IntInfo >> 32 == 0, ("%#RX64\n", u64IntInfo));
8244 Assert(puIntrState);
8245 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
8246
8247 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
8248 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
8249
8250#ifdef VBOX_STRICT
8251 /*
8252 * Validate the error-code-valid bit for hardware exceptions.
8253 * No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling"
8254 */
8255 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8256 && !CPUMIsGuestInRealModeEx(pMixedCtx))
8257 {
8258 switch (uVector)
8259 {
8260 case X86_XCPT_PF:
8261 case X86_XCPT_DF:
8262 case X86_XCPT_TS:
8263 case X86_XCPT_NP:
8264 case X86_XCPT_SS:
8265 case X86_XCPT_GP:
8266 case X86_XCPT_AC:
8267 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
8268 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
8269 RT_FALL_THRU();
8270 default:
8271 break;
8272 }
8273 }
8274#endif
8275
8276 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
8277 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8278 || !(*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
8279
8280 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
8281
8282 /* We require CR0 to check if the guest is in real-mode. */
8283 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
8284 AssertRCReturn(rc, rc);
8285
8286 /*
8287 * Hardware interrupts & exceptions cannot be delivered through the software interrupt redirection bitmap to the real
8288 * mode task in virtual-8086 mode. We must jump to the interrupt handler in the (real-mode) guest.
8289 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode" for interrupt & exception classes.
8290 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
8291 */
8292 if (CPUMIsGuestInRealModeEx(pMixedCtx))
8293 {
8294 PVM pVM = pVCpu->CTX_SUFF(pVM);
8295 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
8296 {
8297 Assert(PDMVmmDevHeapIsEnabled(pVM));
8298 Assert(pVM->hm.s.vmx.pRealModeTSS);
8299
8300 /* We require RIP, RSP, RFLAGS, CS, IDTR. Save the required ones from the VMCS. */
8301 rc = hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
8302 rc |= hmR0VmxSaveGuestTableRegs(pVCpu, pMixedCtx);
8303 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
8304 AssertRCReturn(rc, rc);
8305 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_RIP));
8306
8307 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
8308 size_t const cbIdtEntry = sizeof(X86IDTR16);
8309 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
8310 {
8311 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
8312 if (uVector == X86_XCPT_DF)
8313 return VINF_EM_RESET;
8314
8315 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
8316 if (uVector == X86_XCPT_GP)
8317 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, puIntrState);
8318
8319 /* If we're injecting an interrupt/exception with no valid IDT entry, inject a general-protection fault. */
8320 /* No error codes for exceptions in real-mode. See Intel spec. 20.1.4 "Interrupt and Exception Handling" */
8321 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */,
8322 fStepping, puIntrState);
8323 }
8324
8325 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
8326 uint16_t uGuestIp = pMixedCtx->ip;
8327 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
8328 {
8329 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
8330 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
8331 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8332 }
8333 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
8334 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
8335
8336 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
8337 X86IDTR16 IdtEntry;
8338 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
8339 rc = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
8340 AssertRCReturn(rc, rc);
8341
8342 /* Construct the stack frame for the interrupt/exception handler. */
8343 VBOXSTRICTRC rcStrict;
8344 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
8345 if (rcStrict == VINF_SUCCESS)
8346 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
8347 if (rcStrict == VINF_SUCCESS)
8348 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
8349
8350 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
8351 if (rcStrict == VINF_SUCCESS)
8352 {
8353 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
8354 pMixedCtx->rip = IdtEntry.offSel;
8355 pMixedCtx->cs.Sel = IdtEntry.uSel;
8356 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
8357 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
8358 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8359 && uVector == X86_XCPT_PF)
8360 pMixedCtx->cr2 = GCPtrFaultAddress;
8361
8362 /* If any other guest-state bits are changed here, make sure to update
8363 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
8364 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS
8365 | HM_CHANGED_GUEST_RIP
8366 | HM_CHANGED_GUEST_RFLAGS
8367 | HM_CHANGED_GUEST_RSP);
8368
8369 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
8370 if (*puIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
8371 {
8372 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
8373 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
8374 Log4(("Clearing inhibition due to STI.\n"));
8375 *puIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
8376 }
8377 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
8378 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
8379
8380 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
8381 it, if we are returning to ring-3 before executing guest code. */
8382 pVCpu->hm.s.Event.fPending = false;
8383
8384 /* Make hmR0VmxPreRunGuest return if we're stepping since we've changed cs:rip. */
8385 if (fStepping)
8386 rcStrict = VINF_EM_DBG_STEPPED;
8387 }
8388 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8389 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8390 return rcStrict;
8391 }
8392
8393 /*
8394 * For unrestricted execution enabled CPUs running real-mode guests, we must not set the deliver-error-code bit.
8395 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
8396 */
8397 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
8398 }
8399
8400 /* Validate. */
8401 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
8402 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
8403 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
8404
8405 /* Inject. */
8406 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
8407 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
8408 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
8409 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
8410
8411 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
8412 && uVector == X86_XCPT_PF)
8413 pMixedCtx->cr2 = GCPtrFaultAddress;
8414
8415 Log4(("Injecting vcpu[%RU32] u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x pMixedCtx->uCR2=%#RX64\n", pVCpu->idCpu,
8416 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
8417
8418 AssertRCReturn(rc, rc);
8419 return VINF_SUCCESS;
8420}
8421
8422
8423/**
8424 * Clears the interrupt-window exiting control in the VMCS and if necessary
8425 * clears the current event in the VMCS as well.
8426 *
8427 * @returns VBox status code.
8428 * @param pVCpu The cross context virtual CPU structure.
8429 *
8430 * @remarks Use this function only to clear events that have not yet been
8431 * delivered to the guest but are injected in the VMCS!
8432 * @remarks No-long-jump zone!!!
8433 */
8434static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
8435{
8436 Log4Func(("vcpu[%d]\n", pVCpu->idCpu));
8437
8438 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
8439 hmR0VmxClearIntWindowExitVmcs(pVCpu);
8440
8441 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
8442 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
8443}
8444
8445
8446/**
8447 * Enters the VT-x session.
8448 *
8449 * @returns VBox status code.
8450 * @param pVM The cross context VM structure.
8451 * @param pVCpu The cross context virtual CPU structure.
8452 * @param pCpu Pointer to the CPU info struct.
8453 */
8454VMMR0DECL(int) VMXR0Enter(PVM pVM, PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
8455{
8456 AssertPtr(pVM);
8457 AssertPtr(pVCpu);
8458 Assert(pVM->hm.s.vmx.fSupported);
8459 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8460 NOREF(pCpu); NOREF(pVM);
8461
8462 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8463 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8464
8465#ifdef VBOX_STRICT
8466 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
8467 RTCCUINTREG uHostCR4 = ASMGetCR4();
8468 if (!(uHostCR4 & X86_CR4_VMXE))
8469 {
8470 LogRel(("VMXR0Enter: X86_CR4_VMXE bit in CR4 is not set!\n"));
8471 return VERR_VMX_X86_CR4_VMXE_CLEARED;
8472 }
8473#endif
8474
8475 /*
8476 * Load the VCPU's VMCS as the current (and active) one.
8477 */
8478 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
8479 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8480 if (RT_FAILURE(rc))
8481 return rc;
8482
8483 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8484 pVCpu->hm.s.fLeaveDone = false;
8485 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8486
8487 return VINF_SUCCESS;
8488}
8489
8490
8491/**
8492 * The thread-context callback (only on platforms which support it).
8493 *
8494 * @param enmEvent The thread-context event.
8495 * @param pVCpu The cross context virtual CPU structure.
8496 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
8497 * @thread EMT(pVCpu)
8498 */
8499VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
8500{
8501 NOREF(fGlobalInit);
8502
8503 switch (enmEvent)
8504 {
8505 case RTTHREADCTXEVENT_OUT:
8506 {
8507 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8508 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8509 VMCPU_ASSERT_EMT(pVCpu);
8510
8511 PCPUMCTX pMixedCtx = CPUMQueryGuestCtxPtr(pVCpu);
8512
8513 /* No longjmps (logger flushes, locks) in this fragile context. */
8514 VMMRZCallRing3Disable(pVCpu);
8515 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
8516
8517 /*
8518 * Restore host-state (FPU, debug etc.)
8519 */
8520 if (!pVCpu->hm.s.fLeaveDone)
8521 {
8522 /* Do -not- save guest-state here as we might already be in the middle of saving it (esp. bad if we are
8523 holding the PGM lock while saving the guest state (see hmR0VmxSaveGuestControlRegs()). */
8524 hmR0VmxLeave(pVCpu, pMixedCtx, false /* fSaveGuestState */);
8525 pVCpu->hm.s.fLeaveDone = true;
8526 }
8527
8528 /* Leave HM context, takes care of local init (term). */
8529 int rc = HMR0LeaveCpu(pVCpu);
8530 AssertRC(rc); NOREF(rc);
8531
8532 /* Restore longjmp state. */
8533 VMMRZCallRing3Enable(pVCpu);
8534 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
8535 break;
8536 }
8537
8538 case RTTHREADCTXEVENT_IN:
8539 {
8540 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8541 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
8542 VMCPU_ASSERT_EMT(pVCpu);
8543
8544 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8545 VMMRZCallRing3Disable(pVCpu);
8546 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8547
8548 /* Initialize the bare minimum state required for HM. This takes care of
8549 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8550 int rc = HMR0EnterCpu(pVCpu);
8551 AssertRC(rc);
8552 Assert(HMCPU_CF_IS_SET(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE));
8553
8554 /* Load the active VMCS as the current one. */
8555 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8556 {
8557 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8558 AssertRC(rc); NOREF(rc);
8559 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8560 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8561 }
8562 pVCpu->hm.s.fLeaveDone = false;
8563
8564 /* Restore longjmp state. */
8565 VMMRZCallRing3Enable(pVCpu);
8566 break;
8567 }
8568
8569 default:
8570 break;
8571 }
8572}
8573
8574
8575/**
8576 * Saves the host state in the VMCS host-state.
8577 * Sets up the VM-exit MSR-load area.
8578 *
8579 * The CPU state will be loaded from these fields on every successful VM-exit.
8580 *
8581 * @returns VBox status code.
8582 * @param pVM The cross context VM structure.
8583 * @param pVCpu The cross context virtual CPU structure.
8584 *
8585 * @remarks No-long-jump zone!!!
8586 */
8587static int hmR0VmxSaveHostState(PVM pVM, PVMCPU pVCpu)
8588{
8589 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8590
8591 int rc = VINF_SUCCESS;
8592 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
8593 {
8594 rc = hmR0VmxSaveHostControlRegs(pVM, pVCpu);
8595 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostControlRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8596
8597 rc = hmR0VmxSaveHostSegmentRegs(pVM, pVCpu);
8598 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostSegmentRegisters failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8599
8600 rc = hmR0VmxSaveHostMsrs(pVM, pVCpu);
8601 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSaveHostMsrs failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8602
8603 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_HOST_CONTEXT);
8604 }
8605 return rc;
8606}
8607
8608
8609/**
8610 * Saves the host state in the VMCS host-state.
8611 *
8612 * @returns VBox status code.
8613 * @param pVM The cross context VM structure.
8614 * @param pVCpu The cross context virtual CPU structure.
8615 *
8616 * @remarks No-long-jump zone!!!
8617 */
8618VMMR0DECL(int) VMXR0SaveHostState(PVM pVM, PVMCPU pVCpu)
8619{
8620 AssertPtr(pVM);
8621 AssertPtr(pVCpu);
8622
8623 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8624
8625 /* Save the host state here while entering HM context. When thread-context hooks are used, we might get preempted
8626 and have to resave the host state but most of the time we won't be, so do it here before we disable interrupts. */
8627 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8628 return hmR0VmxSaveHostState(pVM, pVCpu);
8629}
8630
8631
8632/**
8633 * Loads the guest state into the VMCS guest-state area.
8634 *
8635 * The will typically be done before VM-entry when the guest-CPU state and the
8636 * VMCS state may potentially be out of sync.
8637 *
8638 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8639 * VM-entry controls.
8640 * Sets up the appropriate VMX non-root function to execute guest code based on
8641 * the guest CPU mode.
8642 *
8643 * @returns VBox strict status code.
8644 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8645 * without unrestricted guest access and the VMMDev is not presently
8646 * mapped (e.g. EFI32).
8647 *
8648 * @param pVM The cross context VM structure.
8649 * @param pVCpu The cross context virtual CPU structure.
8650 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8651 * out-of-sync. Make sure to update the required fields
8652 * before using them.
8653 *
8654 * @remarks No-long-jump zone!!!
8655 */
8656static VBOXSTRICTRC hmR0VmxLoadGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8657{
8658 AssertPtr(pVM);
8659 AssertPtr(pVCpu);
8660 AssertPtr(pMixedCtx);
8661 HMVMX_ASSERT_PREEMPT_SAFE();
8662
8663 LogFlowFunc(("pVM=%p pVCpu=%p\n", pVM, pVCpu));
8664
8665 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestState, x);
8666
8667 /* Determine real-on-v86 mode. */
8668 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8669 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
8670 && CPUMIsGuestInRealModeEx(pMixedCtx))
8671 {
8672 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8673 }
8674
8675 /*
8676 * Load the guest-state into the VMCS.
8677 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8678 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8679 */
8680 int rc = hmR0VmxSetupVMRunHandler(pVCpu, pMixedCtx);
8681 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupVMRunHandler! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8682
8683 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8684 rc = hmR0VmxLoadGuestEntryCtls(pVCpu, pMixedCtx);
8685 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestEntryCtls! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8686
8687 /* This needs to be done after hmR0VmxSetupVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8688 rc = hmR0VmxLoadGuestExitCtls(pVCpu, pMixedCtx);
8689 AssertLogRelMsgRCReturn(rc, ("hmR0VmxSetupExitCtls failed! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8690
8691 rc = hmR0VmxLoadGuestActivityState(pVCpu, pMixedCtx);
8692 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestActivityState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8693
8694 VBOXSTRICTRC rcStrict = hmR0VmxLoadGuestCR3AndCR4(pVCpu, pMixedCtx);
8695 if (rcStrict == VINF_SUCCESS)
8696 { /* likely */ }
8697 else
8698 {
8699 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8700 return rcStrict;
8701 }
8702
8703 /* Assumes pMixedCtx->cr0 is up-to-date (strict builds require CR0 for segment register validation checks). */
8704 rc = hmR0VmxLoadGuestSegmentRegs(pVCpu, pMixedCtx);
8705 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestSegmentRegs: rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8706
8707 /* This needs to be done after hmR0VmxLoadGuestEntryCtls() and hmR0VmxLoadGuestExitCtls() as it may alter controls if we
8708 determine we don't have to swap EFER after all. */
8709 rc = hmR0VmxLoadGuestMsrs(pVCpu, pMixedCtx);
8710 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestMsrs! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8711
8712 rc = hmR0VmxLoadGuestApicState(pVCpu, pMixedCtx);
8713 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestApicState! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8714
8715 rc = hmR0VmxLoadGuestXcptIntercepts(pVCpu, pMixedCtx);
8716 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestXcptIntercepts! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8717
8718 /*
8719 * Loading Rflags here is fine, even though Rflags.TF might depend on guest debug state (which is not loaded here).
8720 * It is re-evaluated and updated if necessary in hmR0VmxLoadSharedState().
8721 */
8722 rc = hmR0VmxLoadGuestRipRspRflags(pVCpu, pMixedCtx);
8723 AssertLogRelMsgRCReturn(rc, ("hmR0VmxLoadGuestRipRspRflags! rc=%Rrc (pVM=%p pVCpu=%p)\n", rc, pVM, pVCpu), rc);
8724
8725 /* Clear any unused and reserved bits. */
8726 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_GUEST_CR2
8727 | HM_CHANGED_GUEST_HWVIRT);
8728
8729 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestState, x);
8730 return rc;
8731}
8732
8733
8734/**
8735 * Loads the state shared between the host and guest into the VMCS.
8736 *
8737 * @param pVM The cross context VM structure.
8738 * @param pVCpu The cross context virtual CPU structure.
8739 * @param pCtx Pointer to the guest-CPU context.
8740 *
8741 * @remarks No-long-jump zone!!!
8742 */
8743static void hmR0VmxLoadSharedState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
8744{
8745 NOREF(pVM);
8746
8747 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8748 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8749
8750 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_CR0))
8751 {
8752 int rc = hmR0VmxLoadSharedCR0(pVCpu, pCtx);
8753 AssertRC(rc);
8754 }
8755
8756 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_DEBUG))
8757 {
8758 int rc = hmR0VmxLoadSharedDebugState(pVCpu, pCtx);
8759 AssertRC(rc);
8760
8761 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8762 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_GUEST_RFLAGS))
8763 {
8764 rc = hmR0VmxLoadGuestRflags(pVCpu, pCtx);
8765 AssertRC(rc);
8766 }
8767 }
8768
8769 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS))
8770 {
8771 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8772 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
8773 }
8774
8775 /* Loading CR0, debug state might have changed intercepts, update VMCS. */
8776 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS))
8777 {
8778 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
8779 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
8780 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8781 AssertRC(rc);
8782 HMCPU_CF_CLEAR(pVCpu, HM_CHANGED_VMM_GUEST_XCPT_INTERCEPTS);
8783 }
8784
8785 AssertMsg(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE),
8786 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8787}
8788
8789
8790/**
8791 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8792 *
8793 * @returns Strict VBox status code (i.e. informational status codes too).
8794 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8795 * without unrestricted guest access and the VMMDev is not presently
8796 * mapped (e.g. EFI32).
8797 *
8798 * @param pVM The cross context VM structure.
8799 * @param pVCpu The cross context virtual CPU structure.
8800 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8801 * out-of-sync. Make sure to update the required fields
8802 * before using them.
8803 *
8804 * @remarks No-long-jump zone!!!
8805 */
8806static VBOXSTRICTRC hmR0VmxLoadGuestStateOptimal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx)
8807{
8808 HMVMX_ASSERT_PREEMPT_SAFE();
8809 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8810 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8811
8812 Log5(("LoadFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8813#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8814 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
8815#endif
8816
8817 /*
8818 * RIP is what changes the most often and hence if it's the only bit needing to be
8819 * updated, we shall handle it early for performance reasons.
8820 */
8821 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
8822 if (HMCPU_CF_IS_SET_ONLY(pVCpu, HM_CHANGED_GUEST_RIP))
8823 {
8824 rcStrict = hmR0VmxLoadGuestRip(pVCpu, pMixedCtx);
8825 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8826 { /* likely */}
8827 else
8828 {
8829 AssertMsgFailedReturn(("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestRip failed! rc=%Rrc\n",
8830 VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8831 }
8832 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadMinimal);
8833 }
8834 else if (HMCPU_CF_VALUE(pVCpu))
8835 {
8836 rcStrict = hmR0VmxLoadGuestState(pVM, pVCpu, pMixedCtx);
8837 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8838 { /* likely */}
8839 else
8840 {
8841 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM,
8842 ("hmR0VmxLoadGuestStateOptimal: hmR0VmxLoadGuestState failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8843 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8844 return rcStrict;
8845 }
8846 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadFull);
8847 }
8848
8849 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8850 AssertMsg( !HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_ALL_GUEST)
8851 || HMCPU_CF_IS_PENDING_ONLY(pVCpu, HM_CHANGED_HOST_CONTEXT | HM_CHANGED_HOST_GUEST_SHARED_STATE),
8852 ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
8853 return rcStrict;
8854}
8855
8856
8857/**
8858 * Does the preparations before executing guest code in VT-x.
8859 *
8860 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8861 * recompiler/IEM. We must be cautious what we do here regarding committing
8862 * guest-state information into the VMCS assuming we assuredly execute the
8863 * guest in VT-x mode.
8864 *
8865 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8866 * the common-state (TRPM/forceflags), we must undo those changes so that the
8867 * recompiler/IEM can (and should) use them when it resumes guest execution.
8868 * Otherwise such operations must be done when we can no longer exit to ring-3.
8869 *
8870 * @returns Strict VBox status code (i.e. informational status codes too).
8871 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8872 * have been disabled.
8873 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8874 * double-fault into the guest.
8875 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8876 * dispatched directly.
8877 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8878 *
8879 * @param pVM The cross context VM structure.
8880 * @param pVCpu The cross context virtual CPU structure.
8881 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8882 * out-of-sync. Make sure to update the required fields
8883 * before using them.
8884 * @param pVmxTransient Pointer to the VMX transient structure.
8885 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8886 * us ignore some of the reasons for returning to
8887 * ring-3, and return VINF_EM_DBG_STEPPED if event
8888 * dispatching took place.
8889 */
8890static VBOXSTRICTRC hmR0VmxPreRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8891{
8892 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8893
8894#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8895 PGMRZDynMapFlushAutoSet(pVCpu);
8896#endif
8897
8898 /* Check force flag actions that might require us to go back to ring-3. */
8899 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVM, pVCpu, pMixedCtx, fStepping);
8900 if (rcStrict == VINF_SUCCESS)
8901 { /* FFs doesn't get set all the time. */ }
8902 else
8903 return rcStrict;
8904
8905 /*
8906 * Setup the virtualized-APIC accesses.
8907 *
8908 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8909 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8910 *
8911 * This is the reason we do it here and not in hmR0VmxLoadGuestState().
8912 */
8913 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8914 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8915 && PDMHasApic(pVM))
8916 {
8917 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8918 Assert(u64MsrApicBase);
8919 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8920
8921 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8922
8923 /* Unalias any existing mapping. */
8924 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8925 AssertRCReturn(rc, rc);
8926
8927 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8928 Log4(("hmR0VmxPreRunGuest: VCPU%u: Mapped HC APIC-access page at %#RGp\n", pVCpu->idCpu, GCPhysApicBase));
8929 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8930 AssertRCReturn(rc, rc);
8931
8932 /* Update the per-VCPU cache of the APIC base MSR. */
8933 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8934 }
8935
8936 if (TRPMHasTrap(pVCpu))
8937 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8938 uint32_t uIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8939
8940 /*
8941 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus needs to be done with
8942 * longjmps or interrupts + preemption enabled. Event injection might also result in triple-faulting the VM.
8943 */
8944 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, uIntrState, fStepping);
8945 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8946 { /* likely */ }
8947 else
8948 {
8949 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8950 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8951 return rcStrict;
8952 }
8953
8954 /*
8955 * No longjmps to ring-3 from this point on!!!
8956 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8957 * This also disables flushing of the R0-logger instance (if any).
8958 */
8959 VMMRZCallRing3Disable(pVCpu);
8960
8961 /*
8962 * Load the guest state bits.
8963 *
8964 * We cannot perform longjmps while loading the guest state because we do not preserve the
8965 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8966 * CPU migration.
8967 *
8968 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8969 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8970 * Hence, loading of the guest state needs to be done -after- injection of events.
8971 */
8972 rcStrict = hmR0VmxLoadGuestStateOptimal(pVM, pVCpu, pMixedCtx);
8973 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8974 { /* likely */ }
8975 else
8976 {
8977 VMMRZCallRing3Enable(pVCpu);
8978 return rcStrict;
8979 }
8980
8981 /*
8982 * We disable interrupts so that we don't miss any interrupts that would flag preemption (IPI/timers etc.)
8983 * when thread-context hooks aren't used and we've been running with preemption disabled for a while.
8984 *
8985 * We need to check for force-flags that could've possible been altered since we last checked them (e.g.
8986 * by PDMGetInterrupt() leaving the PDM critical section, see @bugref{6398}).
8987 *
8988 * We also check a couple of other force-flags as a last opportunity to get the EMT back to ring-3 before
8989 * executing guest code.
8990 */
8991 pVmxTransient->fEFlags = ASMIntDisableFlags();
8992
8993 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8994 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8995 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8996 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8997 {
8998 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8999 {
9000 pVCpu->hm.s.Event.fPending = false;
9001
9002 /*
9003 * We've injected any pending events. This is really the point of no return (to ring-3).
9004 *
9005 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
9006 * returns from this function, so don't enable them here.
9007 */
9008 return VINF_SUCCESS;
9009 }
9010
9011 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
9012 rcStrict = VINF_EM_RAW_INTERRUPT;
9013 }
9014 else
9015 {
9016 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
9017 rcStrict = VINF_EM_RAW_TO_R3;
9018 }
9019
9020 ASMSetFlags(pVmxTransient->fEFlags);
9021 VMMRZCallRing3Enable(pVCpu);
9022
9023 return rcStrict;
9024}
9025
9026
9027/**
9028 * Prepares to run guest code in VT-x and we've committed to doing so. This
9029 * means there is no backing out to ring-3 or anywhere else at this
9030 * point.
9031 *
9032 * @param pVM The cross context VM structure.
9033 * @param pVCpu The cross context virtual CPU structure.
9034 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9035 * out-of-sync. Make sure to update the required fields
9036 * before using them.
9037 * @param pVmxTransient Pointer to the VMX transient structure.
9038 *
9039 * @remarks Called with preemption disabled.
9040 * @remarks No-long-jump zone!!!
9041 */
9042static void hmR0VmxPreRunGuestCommitted(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
9043{
9044 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9045 Assert(VMMR0IsLogFlushDisabled(pVCpu));
9046 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
9047
9048 /*
9049 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
9050 */
9051 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9052 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
9053
9054 if (!CPUMIsGuestFPUStateActive(pVCpu))
9055 {
9056 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9057 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
9058 HMCPU_CF_SET(pVCpu, HM_CHANGED_HOST_CONTEXT);
9059 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
9060 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
9061 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_CR0));
9062 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9063 }
9064
9065 /*
9066 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
9067 */
9068 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
9069 && pVCpu->hm.s.vmx.cMsrs > 0)
9070 {
9071 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
9072 }
9073
9074 /*
9075 * Load the host state bits as we may've been preempted (only happens when
9076 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
9077 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
9078 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
9079 * See @bugref{8432}.
9080 */
9081 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT))
9082 {
9083 int rc = hmR0VmxSaveHostState(pVM, pVCpu);
9084 AssertRC(rc);
9085 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptSaveHostState);
9086 }
9087 Assert(!HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_CONTEXT));
9088
9089 /*
9090 * Load the state shared between host and guest (FPU, debug, lazy MSRs).
9091 */
9092 if (HMCPU_CF_IS_PENDING(pVCpu, HM_CHANGED_HOST_GUEST_SHARED_STATE))
9093 hmR0VmxLoadSharedState(pVM, pVCpu, pMixedCtx);
9094 AssertMsg(!HMCPU_CF_VALUE(pVCpu), ("fContextUseFlags=%#RX32\n", HMCPU_CF_VALUE(pVCpu)));
9095
9096 /* Store status of the shared guest-host state at the time of VM-entry. */
9097#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
9098 if (CPUMIsGuestInLongModeEx(pMixedCtx))
9099 {
9100 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
9101 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
9102 }
9103 else
9104#endif
9105 {
9106 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
9107 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
9108 }
9109
9110 /*
9111 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
9112 */
9113 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9114 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
9115
9116 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
9117 RTCPUID idCurrentCpu = pCpu->idCpu;
9118 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
9119 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
9120 {
9121 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVM, pVCpu);
9122 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
9123 }
9124
9125 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
9126 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
9127 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
9128 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
9129
9130 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
9131
9132 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
9133 to start executing. */
9134
9135 /*
9136 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
9137 */
9138 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
9139 {
9140 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9141 {
9142 bool fMsrUpdated;
9143 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9144 AssertRC(rc2);
9145 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9146
9147 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
9148 &fMsrUpdated);
9149 AssertRC(rc2);
9150 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9151
9152 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9153 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9154 }
9155 else
9156 {
9157 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
9158 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9159 }
9160 }
9161
9162 if (pVM->cpum.ro.GuestFeatures.fIbrs)
9163 {
9164 bool fMsrUpdated;
9165 int rc2 = hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
9166 AssertRC(rc2);
9167 Assert(HMVMXCPU_GST_IS_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_AUTO_LOAD_STORE_MSRS));
9168
9169 rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
9170 &fMsrUpdated);
9171 AssertRC(rc2);
9172 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
9173 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
9174 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
9175 }
9176
9177#ifdef VBOX_STRICT
9178 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
9179 hmR0VmxCheckHostEferMsr(pVCpu);
9180 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
9181#endif
9182#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
9183 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVM, pVCpu, pMixedCtx);
9184 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
9185 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
9186#endif
9187}
9188
9189
9190/**
9191 * Performs some essential restoration of state after running guest code in
9192 * VT-x.
9193 *
9194 * @param pVM The cross context VM structure.
9195 * @param pVCpu The cross context virtual CPU structure.
9196 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
9197 * out-of-sync. Make sure to update the required fields
9198 * before using them.
9199 * @param pVmxTransient Pointer to the VMX transient structure.
9200 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
9201 *
9202 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
9203 *
9204 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
9205 * unconditionally when it is safe to do so.
9206 */
9207static void hmR0VmxPostRunGuest(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, int rcVMRun)
9208{
9209 NOREF(pVM);
9210
9211 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
9212
9213 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
9214 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
9215 HMVMXCPU_GST_RESET_TO(pVCpu, 0); /* Exits/longjmps to ring-3 requires saving the guest state. */
9216 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
9217 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
9218 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
9219
9220 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9221 TMCpuTickSetLastSeen(pVCpu, ASMReadTSC() + pVCpu->hm.s.vmx.u64TSCOffset);
9222
9223 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatExit1, x);
9224 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
9225 Assert(!ASMIntAreEnabled());
9226 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
9227
9228#ifdef HMVMX_ALWAYS_SWAP_FPU_STATE
9229 if (CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVM, pVCpu))
9230 {
9231 hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
9232 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9233 }
9234#endif
9235
9236#if HC_ARCH_BITS == 64
9237 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
9238#endif
9239#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
9240 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
9241 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
9242 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9243#else
9244 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
9245#endif
9246#ifdef VBOX_STRICT
9247 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
9248#endif
9249 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
9250
9251 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
9252 uint32_t uExitReason;
9253 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
9254 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
9255 AssertRC(rc);
9256 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
9257 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
9258
9259 if (rcVMRun == VINF_SUCCESS)
9260 {
9261 /*
9262 * Update the VM-exit history array here even if the VM-entry failed due to:
9263 * - Invalid guest state.
9264 * - MSR loading.
9265 * - Machine-check event.
9266 *
9267 * In any of the above cases we will still have a "valid" VM-exit reason
9268 * despite @a fVMEntryFailed being false.
9269 *
9270 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
9271 */
9272 HMCPU_EXIT_HISTORY_ADD(pVCpu, pVmxTransient->uExitReason);
9273
9274 if (!pVmxTransient->fVMEntryFailed)
9275 {
9276 /** @todo We can optimize this by only syncing with our force-flags when
9277 * really needed and keeping the VMCS state as it is for most
9278 * VM-exits. */
9279 /* Update the guest interruptibility-state from the VMCS. */
9280 hmR0VmxSaveGuestIntrState(pVCpu, pMixedCtx);
9281
9282 /*
9283 * Allow longjmps to ring-3 -after- saving the guest-interruptibility state
9284 * as it's not part of hmR0VmxSaveGuestState() and thus would trigger an assertion
9285 * on the longjmp path to ring-3 while saving the (rest of) the guest state,
9286 * see @bugref{6208#c63}.
9287 */
9288 VMMRZCallRing3Enable(pVCpu);
9289
9290#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
9291 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
9292 AssertRC(rc);
9293#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
9294 rc = hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
9295 AssertRC(rc);
9296#endif
9297
9298 /*
9299 * Sync the TPR shadow with our APIC state.
9300 */
9301 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
9302 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
9303 {
9304 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
9305 AssertRC(rc);
9306 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
9307 }
9308
9309 return;
9310 }
9311 }
9312 else
9313 {
9314 Log4(("VM-entry failure: pVCpu=%p idCpu=%RU32 rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", pVCpu, pVCpu->idCpu, rcVMRun,
9315 pVmxTransient->fVMEntryFailed));
9316 }
9317
9318 VMMRZCallRing3Enable(pVCpu);
9319}
9320
9321
9322/**
9323 * Runs the guest code using VT-x the normal way.
9324 *
9325 * @returns VBox status code.
9326 * @param pVM The cross context VM structure.
9327 * @param pVCpu The cross context virtual CPU structure.
9328 * @param pCtx Pointer to the guest-CPU context.
9329 *
9330 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
9331 */
9332static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
9333{
9334 VMXTRANSIENT VmxTransient;
9335 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9336 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9337 uint32_t cLoops = 0;
9338
9339 for (;; cLoops++)
9340 {
9341 Assert(!HMR0SuspendPending());
9342 HMVMX_ASSERT_CPU_SAFE();
9343
9344 /* Preparatory work for running guest code, this may force us to return
9345 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
9346 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9347 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, false /* fStepping */);
9348 if (rcStrict != VINF_SUCCESS)
9349 break;
9350
9351 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
9352 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
9353 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9354
9355 /* Restore any residual host-state and save any bits shared between host
9356 and guest into the guest-CPU state. Re-enables interrupts! */
9357 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
9358
9359 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9360 if (RT_SUCCESS(rcRun))
9361 { /* very likely */ }
9362 else
9363 {
9364 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
9365 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
9366 return rcRun;
9367 }
9368
9369 /* Profile the VM-exit. */
9370 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9371 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9372 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9373 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
9374 HMVMX_START_EXIT_DISPATCH_PROF();
9375
9376 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9377
9378 /* Handle the VM-exit. */
9379#ifdef HMVMX_USE_FUNCTION_TABLE
9380 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
9381#else
9382 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
9383#endif
9384 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
9385 if (rcStrict == VINF_SUCCESS)
9386 {
9387 if (cLoops <= pVM->hm.s.cMaxResumeLoops)
9388 continue; /* likely */
9389 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9390 rcStrict = VINF_EM_RAW_INTERRUPT;
9391 }
9392 break;
9393 }
9394
9395 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9396 return rcStrict;
9397}
9398
9399
9400
9401/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
9402 * probes.
9403 *
9404 * The following few functions and associated structure contains the bloat
9405 * necessary for providing detailed debug events and dtrace probes as well as
9406 * reliable host side single stepping. This works on the principle of
9407 * "subclassing" the normal execution loop and workers. We replace the loop
9408 * method completely and override selected helpers to add necessary adjustments
9409 * to their core operation.
9410 *
9411 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
9412 * any performance for debug and analysis features.
9413 *
9414 * @{
9415 */
9416
9417/**
9418 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
9419 * the debug run loop.
9420 */
9421typedef struct VMXRUNDBGSTATE
9422{
9423 /** The RIP we started executing at. This is for detecting that we stepped. */
9424 uint64_t uRipStart;
9425 /** The CS we started executing with. */
9426 uint16_t uCsStart;
9427
9428 /** Whether we've actually modified the 1st execution control field. */
9429 bool fModifiedProcCtls : 1;
9430 /** Whether we've actually modified the 2nd execution control field. */
9431 bool fModifiedProcCtls2 : 1;
9432 /** Whether we've actually modified the exception bitmap. */
9433 bool fModifiedXcptBitmap : 1;
9434
9435 /** We desire the modified the CR0 mask to be cleared. */
9436 bool fClearCr0Mask : 1;
9437 /** We desire the modified the CR4 mask to be cleared. */
9438 bool fClearCr4Mask : 1;
9439 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
9440 uint32_t fCpe1Extra;
9441 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
9442 uint32_t fCpe1Unwanted;
9443 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
9444 uint32_t fCpe2Extra;
9445 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
9446 uint32_t bmXcptExtra;
9447 /** The sequence number of the Dtrace provider settings the state was
9448 * configured against. */
9449 uint32_t uDtraceSettingsSeqNo;
9450 /** VM-exits to check (one bit per VM-exit). */
9451 uint32_t bmExitsToCheck[3];
9452
9453 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
9454 uint32_t fProcCtlsInitial;
9455 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
9456 uint32_t fProcCtls2Initial;
9457 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
9458 uint32_t bmXcptInitial;
9459} VMXRUNDBGSTATE;
9460AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
9461typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
9462
9463
9464/**
9465 * Initializes the VMXRUNDBGSTATE structure.
9466 *
9467 * @param pVCpu The cross context virtual CPU structure of the
9468 * calling EMT.
9469 * @param pCtx The CPU register context to go with @a pVCpu.
9470 * @param pDbgState The structure to initialize.
9471 */
9472DECLINLINE(void) hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
9473{
9474 pDbgState->uRipStart = pCtx->rip;
9475 pDbgState->uCsStart = pCtx->cs.Sel;
9476
9477 pDbgState->fModifiedProcCtls = false;
9478 pDbgState->fModifiedProcCtls2 = false;
9479 pDbgState->fModifiedXcptBitmap = false;
9480 pDbgState->fClearCr0Mask = false;
9481 pDbgState->fClearCr4Mask = false;
9482 pDbgState->fCpe1Extra = 0;
9483 pDbgState->fCpe1Unwanted = 0;
9484 pDbgState->fCpe2Extra = 0;
9485 pDbgState->bmXcptExtra = 0;
9486 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
9487 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
9488 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
9489}
9490
9491
9492/**
9493 * Updates the VMSC fields with changes requested by @a pDbgState.
9494 *
9495 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
9496 * immediately before executing guest code, i.e. when interrupts are disabled.
9497 * We don't check status codes here as we cannot easily assert or return in the
9498 * latter case.
9499 *
9500 * @param pVCpu The cross context virtual CPU structure.
9501 * @param pDbgState The debug state.
9502 */
9503DECLINLINE(void) hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
9504{
9505 /*
9506 * Ensure desired flags in VMCS control fields are set.
9507 * (Ignoring write failure here, as we're committed and it's just debug extras.)
9508 *
9509 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
9510 * there should be no stale data in pCtx at this point.
9511 */
9512 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
9513 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
9514 {
9515 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
9516 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
9517 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
9518 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
9519 pDbgState->fModifiedProcCtls = true;
9520 }
9521
9522 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
9523 {
9524 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
9525 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
9526 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
9527 pDbgState->fModifiedProcCtls2 = true;
9528 }
9529
9530 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
9531 {
9532 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
9533 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
9534 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
9535 pDbgState->fModifiedXcptBitmap = true;
9536 }
9537
9538 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
9539 {
9540 pVCpu->hm.s.vmx.u32CR0Mask = 0;
9541 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
9542 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
9543 }
9544
9545 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
9546 {
9547 pVCpu->hm.s.vmx.u32CR4Mask = 0;
9548 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9549 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9550 }
9551}
9552
9553
9554DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9555{
9556 /*
9557 * Restore VM-exit control settings as we may not reenter this function the
9558 * next time around.
9559 */
9560 /* We reload the initial value, trigger what we can of recalculations the
9561 next time around. From the looks of things, that's all that's required atm. */
9562 if (pDbgState->fModifiedProcCtls)
9563 {
9564 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9565 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9566 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9567 AssertRCReturn(rc2, rc2);
9568 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9569 }
9570
9571 /* We're currently the only ones messing with this one, so just restore the
9572 cached value and reload the field. */
9573 if ( pDbgState->fModifiedProcCtls2
9574 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9575 {
9576 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9577 AssertRCReturn(rc2, rc2);
9578 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9579 }
9580
9581 /* If we've modified the exception bitmap, we restore it and trigger
9582 reloading and partial recalculation the next time around. */
9583 if (pDbgState->fModifiedXcptBitmap)
9584 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9585
9586 return rcStrict;
9587}
9588
9589
9590/**
9591 * Configures VM-exit controls for current DBGF and DTrace settings.
9592 *
9593 * This updates @a pDbgState and the VMCS execution control fields to reflect
9594 * the necessary VM-exits demanded by DBGF and DTrace.
9595 *
9596 * @param pVM The cross context VM structure.
9597 * @param pVCpu The cross context virtual CPU structure.
9598 * @param pCtx Pointer to the guest-CPU context.
9599 * @param pDbgState The debug state.
9600 * @param pVmxTransient Pointer to the VMX transient structure. May update
9601 * fUpdateTscOffsettingAndPreemptTimer.
9602 */
9603static void hmR0VmxPreRunGuestDebugStateUpdate(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx,
9604 PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9605{
9606 /*
9607 * Take down the dtrace serial number so we can spot changes.
9608 */
9609 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9610 ASMCompilerBarrier();
9611
9612 /*
9613 * We'll rebuild most of the middle block of data members (holding the
9614 * current settings) as we go along here, so start by clearing it all.
9615 */
9616 pDbgState->bmXcptExtra = 0;
9617 pDbgState->fCpe1Extra = 0;
9618 pDbgState->fCpe1Unwanted = 0;
9619 pDbgState->fCpe2Extra = 0;
9620 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9621 pDbgState->bmExitsToCheck[i] = 0;
9622
9623 /*
9624 * Software interrupts (INT XXh) - no idea how to trigger these...
9625 */
9626 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9627 || VBOXVMM_INT_SOFTWARE_ENABLED())
9628 {
9629 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9630 }
9631
9632 /*
9633 * INT3 breakpoints - triggered by #BP exceptions.
9634 */
9635 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9636 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9637
9638 /*
9639 * Exception bitmap and XCPT events+probes.
9640 */
9641 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9642 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9643 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9644
9645 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9646 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9647 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9648 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9649 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9650 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9651 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9652 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9653 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9654 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9655 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9656 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9657 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9658 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9659 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9660 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9661 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9662 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9663
9664 if (pDbgState->bmXcptExtra)
9665 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9666
9667 /*
9668 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9669 *
9670 * Note! This is the reverse of waft hmR0VmxHandleExitDtraceEvents does.
9671 * So, when adding/changing/removing please don't forget to update it.
9672 *
9673 * Some of the macros are picking up local variables to save horizontal space,
9674 * (being able to see it in a table is the lesser evil here).
9675 */
9676#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9677 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9678 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9679#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9680 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9681 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9682 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9683 } else do { } while (0)
9684#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9685 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9686 { \
9687 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9688 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9689 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9690 } else do { } while (0)
9691#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9692 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9693 { \
9694 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9695 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9696 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9697 } else do { } while (0)
9698#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9699 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9700 { \
9701 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9702 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9703 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9704 } else do { } while (0)
9705
9706 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9707 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9708 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9709 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9710 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9711
9712 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9713 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9714 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9715 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9716 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9717 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9718 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9719 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9720 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9721 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9722 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9723 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9724 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9725 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9726 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9727 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9728 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9729 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9730 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9731 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9732 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9733 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9734 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9735 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9736 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9737 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9738 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9739 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9740 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9741 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9742 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9743 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9744 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9745 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9746 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9747 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9748
9749 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9750 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9751 {
9752 int rc2 = hmR0VmxSaveGuestCR0(pVCpu, pCtx);
9753 rc2 |= hmR0VmxSaveGuestCR4(pVCpu, pCtx);
9754 rc2 |= hmR0VmxSaveGuestApicState(pVCpu, pCtx);
9755 AssertRC(rc2);
9756
9757#if 0 /** @todo fix me */
9758 pDbgState->fClearCr0Mask = true;
9759 pDbgState->fClearCr4Mask = true;
9760#endif
9761 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9762 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9763 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9764 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9765 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9766 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9767 require clearing here and in the loop if we start using it. */
9768 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9769 }
9770 else
9771 {
9772 if (pDbgState->fClearCr0Mask)
9773 {
9774 pDbgState->fClearCr0Mask = false;
9775 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
9776 }
9777 if (pDbgState->fClearCr4Mask)
9778 {
9779 pDbgState->fClearCr4Mask = false;
9780 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
9781 }
9782 }
9783 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9784 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9785
9786 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9787 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9788 {
9789 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9790 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9791 }
9792 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9793 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9794
9795 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9796 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9797 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9798 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9799 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9800 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9801 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9802 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9803#if 0 /** @todo too slow, fix handler. */
9804 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9805#endif
9806 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9807
9808 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9809 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9810 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9811 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9812 {
9813 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9814 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9815 }
9816 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9817 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9818 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9819 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9820
9821 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9822 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9823 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9824 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9825 {
9826 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9827 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9828 }
9829 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9830 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9831 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9832 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9833
9834 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9835 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9836 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9837 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9838 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9839 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9840 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9841 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9842 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9843 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9844 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9845 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9846 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9847 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9848 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9849 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9850 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9851 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9852 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9853 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9854 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9855 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9856
9857#undef IS_EITHER_ENABLED
9858#undef SET_ONLY_XBM_IF_EITHER_EN
9859#undef SET_CPE1_XBM_IF_EITHER_EN
9860#undef SET_CPEU_XBM_IF_EITHER_EN
9861#undef SET_CPE2_XBM_IF_EITHER_EN
9862
9863 /*
9864 * Sanitize the control stuff.
9865 */
9866 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9867 if (pDbgState->fCpe2Extra)
9868 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9869 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9870 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9871 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9872 {
9873 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9874 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9875 }
9876
9877 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9878 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9879 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9880 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9881}
9882
9883
9884/**
9885 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9886 * appropriate.
9887 *
9888 * The caller has checked the VM-exit against the
9889 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9890 * already, so we don't have to do that either.
9891 *
9892 * @returns Strict VBox status code (i.e. informational status codes too).
9893 * @param pVM The cross context VM structure.
9894 * @param pVCpu The cross context virtual CPU structure.
9895 * @param pMixedCtx Pointer to the guest-CPU context.
9896 * @param pVmxTransient Pointer to the VMX-transient structure.
9897 * @param uExitReason The VM-exit reason.
9898 *
9899 * @remarks The name of this function is displayed by dtrace, so keep it short
9900 * and to the point. No longer than 33 chars long, please.
9901 */
9902static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx,
9903 PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9904{
9905 /*
9906 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9907 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9908 *
9909 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9910 * does. Must add/change/remove both places. Same ordering, please.
9911 *
9912 * Added/removed events must also be reflected in the next section
9913 * where we dispatch dtrace events.
9914 */
9915 bool fDtrace1 = false;
9916 bool fDtrace2 = false;
9917 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9918 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9919 uint32_t uEventArg = 0;
9920#define SET_EXIT(a_EventSubName) \
9921 do { \
9922 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9923 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9924 } while (0)
9925#define SET_BOTH(a_EventSubName) \
9926 do { \
9927 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9928 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9929 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9930 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9931 } while (0)
9932 switch (uExitReason)
9933 {
9934 case VMX_EXIT_MTF:
9935 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9936
9937 case VMX_EXIT_XCPT_OR_NMI:
9938 {
9939 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9940 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9941 {
9942 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9943 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9944 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9945 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9946 {
9947 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9948 {
9949 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9950 uEventArg = pVmxTransient->uExitIntErrorCode;
9951 }
9952 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9953 switch (enmEvent1)
9954 {
9955 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9956 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9957 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9958 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9959 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9960 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9961 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9962 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9963 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9964 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9965 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9966 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9967 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9968 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9969 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9970 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9971 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9972 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9973 default: break;
9974 }
9975 }
9976 else
9977 AssertFailed();
9978 break;
9979
9980 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9981 uEventArg = idxVector;
9982 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9983 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9984 break;
9985 }
9986 break;
9987 }
9988
9989 case VMX_EXIT_TRIPLE_FAULT:
9990 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9991 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9992 break;
9993 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9994 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9995 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9996 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9997 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9998
9999 /* Instruction specific VM-exits: */
10000 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
10001 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
10002 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
10003 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
10004 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
10005 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
10006 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
10007 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
10008 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
10009 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
10010 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
10011 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
10012 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
10013 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
10014 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
10015 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
10016 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
10017 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
10018 case VMX_EXIT_MOV_CRX:
10019 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10020/** @todo r=bird: I feel these macros aren't very descriptive and needs to be at least 30 chars longer! ;-)
10021* Sensible abbreviations strongly recommended here because even with 130 columns this stuff get too wide! */
10022 if ( VMX_EXIT_QUALIFICATION_CRX_ACCESS(pVmxTransient->uExitQualification)
10023 == VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ)
10024 SET_BOTH(CRX_READ);
10025 else
10026 SET_BOTH(CRX_WRITE);
10027 uEventArg = VMX_EXIT_QUALIFICATION_CRX_REGISTER(pVmxTransient->uExitQualification);
10028 break;
10029 case VMX_EXIT_MOV_DRX:
10030 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10031 if ( VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification)
10032 == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_READ)
10033 SET_BOTH(DRX_READ);
10034 else
10035 SET_BOTH(DRX_WRITE);
10036 uEventArg = VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification);
10037 break;
10038 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
10039 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
10040 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
10041 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
10042 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
10043 case VMX_EXIT_XDTR_ACCESS:
10044 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10045 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
10046 {
10047 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
10048 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
10049 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
10050 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
10051 }
10052 break;
10053
10054 case VMX_EXIT_TR_ACCESS:
10055 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
10056 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
10057 {
10058 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
10059 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
10060 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
10061 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
10062 }
10063 break;
10064
10065 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
10066 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
10067 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
10068 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
10069 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
10070 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
10071 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
10072 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
10073 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
10074 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
10075 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
10076
10077 /* Events that aren't relevant at this point. */
10078 case VMX_EXIT_EXT_INT:
10079 case VMX_EXIT_INT_WINDOW:
10080 case VMX_EXIT_NMI_WINDOW:
10081 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10082 case VMX_EXIT_PREEMPT_TIMER:
10083 case VMX_EXIT_IO_INSTR:
10084 break;
10085
10086 /* Errors and unexpected events. */
10087 case VMX_EXIT_INIT_SIGNAL:
10088 case VMX_EXIT_SIPI:
10089 case VMX_EXIT_IO_SMI:
10090 case VMX_EXIT_SMI:
10091 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10092 case VMX_EXIT_ERR_MSR_LOAD:
10093 case VMX_EXIT_ERR_MACHINE_CHECK:
10094 break;
10095
10096 default:
10097 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10098 break;
10099 }
10100#undef SET_BOTH
10101#undef SET_EXIT
10102
10103 /*
10104 * Dtrace tracepoints go first. We do them here at once so we don't
10105 * have to copy the guest state saving and stuff a few dozen times.
10106 * Down side is that we've got to repeat the switch, though this time
10107 * we use enmEvent since the probes are a subset of what DBGF does.
10108 */
10109 if (fDtrace1 || fDtrace2)
10110 {
10111 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10112 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10113 switch (enmEvent1)
10114 {
10115 /** @todo consider which extra parameters would be helpful for each probe. */
10116 case DBGFEVENT_END: break;
10117 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
10118 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
10119 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
10120 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
10121 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
10122 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
10123 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
10124 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
10125 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
10126 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
10127 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
10128 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
10129 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
10130 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
10131 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
10132 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
10133 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
10134 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
10135 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10136 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10137 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
10138 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
10139 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
10140 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
10141 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
10142 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
10143 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
10144 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10145 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10146 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10147 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10148 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10149 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10150 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10151 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
10152 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
10153 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
10154 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
10155 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
10156 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
10157 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
10158 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
10159 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
10160 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
10161 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
10162 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
10163 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
10164 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
10165 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
10166 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
10167 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
10168 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
10169 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
10170 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10171 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10172 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10173 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10174 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
10175 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10176 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10177 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10178 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
10179 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
10180 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
10181 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
10182 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10183 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
10184 }
10185 switch (enmEvent2)
10186 {
10187 /** @todo consider which extra parameters would be helpful for each probe. */
10188 case DBGFEVENT_END: break;
10189 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
10190 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
10191 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
10192 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
10193 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
10194 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
10195 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
10196 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
10197 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
10198 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10199 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10200 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10201 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
10202 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
10203 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
10204 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
10205 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
10206 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
10207 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
10208 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
10209 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
10210 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
10211 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
10212 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
10213 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
10214 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
10215 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
10216 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
10217 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
10218 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
10219 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
10220 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
10221 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
10222 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
10223 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
10224 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
10225 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
10226 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
10227 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
10228 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
10229 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
10230 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
10231 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
10232 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
10233 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
10234 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
10235 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
10236 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
10237 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
10238 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
10239 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
10240 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
10241 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
10242 }
10243 }
10244
10245 /*
10246 * Fire of the DBGF event, if enabled (our check here is just a quick one,
10247 * the DBGF call will do a full check).
10248 *
10249 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
10250 * Note! If we have to events, we prioritize the first, i.e. the instruction
10251 * one, in order to avoid event nesting.
10252 */
10253 if ( enmEvent1 != DBGFEVENT_END
10254 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
10255 {
10256 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
10257 if (rcStrict != VINF_SUCCESS)
10258 return rcStrict;
10259 }
10260 else if ( enmEvent2 != DBGFEVENT_END
10261 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
10262 {
10263 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
10264 if (rcStrict != VINF_SUCCESS)
10265 return rcStrict;
10266 }
10267
10268 return VINF_SUCCESS;
10269}
10270
10271
10272/**
10273 * Single-stepping VM-exit filtering.
10274 *
10275 * This is preprocessing the VM-exits and deciding whether we've gotten far
10276 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
10277 * handling is performed.
10278 *
10279 * @returns Strict VBox status code (i.e. informational status codes too).
10280 * @param pVM The cross context VM structure.
10281 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
10282 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
10283 * out-of-sync. Make sure to update the required
10284 * fields before using them.
10285 * @param pVmxTransient Pointer to the VMX-transient structure.
10286 * @param uExitReason The VM-exit reason.
10287 * @param pDbgState The debug state.
10288 */
10289DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVM pVM, PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
10290 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
10291{
10292 /*
10293 * Expensive (saves context) generic dtrace VM-exit probe.
10294 */
10295 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
10296 { /* more likely */ }
10297 else
10298 {
10299 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
10300 hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
10301 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
10302 }
10303
10304 /*
10305 * Check for host NMI, just to get that out of the way.
10306 */
10307 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
10308 { /* normally likely */ }
10309 else
10310 {
10311 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10312 AssertRCReturn(rc2, rc2);
10313 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10314 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10315 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
10316 }
10317
10318 /*
10319 * Check for single stepping event if we're stepping.
10320 */
10321 if (pVCpu->hm.s.fSingleInstruction)
10322 {
10323 switch (uExitReason)
10324 {
10325 case VMX_EXIT_MTF:
10326 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
10327
10328 /* Various events: */
10329 case VMX_EXIT_XCPT_OR_NMI:
10330 case VMX_EXIT_EXT_INT:
10331 case VMX_EXIT_TRIPLE_FAULT:
10332 case VMX_EXIT_INT_WINDOW:
10333 case VMX_EXIT_NMI_WINDOW:
10334 case VMX_EXIT_TASK_SWITCH:
10335 case VMX_EXIT_TPR_BELOW_THRESHOLD:
10336 case VMX_EXIT_APIC_ACCESS:
10337 case VMX_EXIT_EPT_VIOLATION:
10338 case VMX_EXIT_EPT_MISCONFIG:
10339 case VMX_EXIT_PREEMPT_TIMER:
10340
10341 /* Instruction specific VM-exits: */
10342 case VMX_EXIT_CPUID:
10343 case VMX_EXIT_GETSEC:
10344 case VMX_EXIT_HLT:
10345 case VMX_EXIT_INVD:
10346 case VMX_EXIT_INVLPG:
10347 case VMX_EXIT_RDPMC:
10348 case VMX_EXIT_RDTSC:
10349 case VMX_EXIT_RSM:
10350 case VMX_EXIT_VMCALL:
10351 case VMX_EXIT_VMCLEAR:
10352 case VMX_EXIT_VMLAUNCH:
10353 case VMX_EXIT_VMPTRLD:
10354 case VMX_EXIT_VMPTRST:
10355 case VMX_EXIT_VMREAD:
10356 case VMX_EXIT_VMRESUME:
10357 case VMX_EXIT_VMWRITE:
10358 case VMX_EXIT_VMXOFF:
10359 case VMX_EXIT_VMXON:
10360 case VMX_EXIT_MOV_CRX:
10361 case VMX_EXIT_MOV_DRX:
10362 case VMX_EXIT_IO_INSTR:
10363 case VMX_EXIT_RDMSR:
10364 case VMX_EXIT_WRMSR:
10365 case VMX_EXIT_MWAIT:
10366 case VMX_EXIT_MONITOR:
10367 case VMX_EXIT_PAUSE:
10368 case VMX_EXIT_XDTR_ACCESS:
10369 case VMX_EXIT_TR_ACCESS:
10370 case VMX_EXIT_INVEPT:
10371 case VMX_EXIT_RDTSCP:
10372 case VMX_EXIT_INVVPID:
10373 case VMX_EXIT_WBINVD:
10374 case VMX_EXIT_XSETBV:
10375 case VMX_EXIT_RDRAND:
10376 case VMX_EXIT_INVPCID:
10377 case VMX_EXIT_VMFUNC:
10378 case VMX_EXIT_RDSEED:
10379 case VMX_EXIT_XSAVES:
10380 case VMX_EXIT_XRSTORS:
10381 {
10382 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10383 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
10384 AssertRCReturn(rc2, rc2);
10385 if ( pMixedCtx->rip != pDbgState->uRipStart
10386 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
10387 return VINF_EM_DBG_STEPPED;
10388 break;
10389 }
10390
10391 /* Errors and unexpected events: */
10392 case VMX_EXIT_INIT_SIGNAL:
10393 case VMX_EXIT_SIPI:
10394 case VMX_EXIT_IO_SMI:
10395 case VMX_EXIT_SMI:
10396 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
10397 case VMX_EXIT_ERR_MSR_LOAD:
10398 case VMX_EXIT_ERR_MACHINE_CHECK:
10399 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
10400 break;
10401
10402 default:
10403 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
10404 break;
10405 }
10406 }
10407
10408 /*
10409 * Check for debugger event breakpoints and dtrace probes.
10410 */
10411 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
10412 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
10413 {
10414 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVM, pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10415 if (rcStrict != VINF_SUCCESS)
10416 return rcStrict;
10417 }
10418
10419 /*
10420 * Normal processing.
10421 */
10422#ifdef HMVMX_USE_FUNCTION_TABLE
10423 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
10424#else
10425 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
10426#endif
10427}
10428
10429
10430/**
10431 * Single steps guest code using VT-x.
10432 *
10433 * @returns Strict VBox status code (i.e. informational status codes too).
10434 * @param pVM The cross context VM structure.
10435 * @param pVCpu The cross context virtual CPU structure.
10436 * @param pCtx Pointer to the guest-CPU context.
10437 *
10438 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
10439 */
10440static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10441{
10442 VMXTRANSIENT VmxTransient;
10443 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
10444
10445 /* Set HMCPU indicators. */
10446 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
10447 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
10448 pVCpu->hm.s.fDebugWantRdTscExit = false;
10449 pVCpu->hm.s.fUsingDebugLoop = true;
10450
10451 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
10452 VMXRUNDBGSTATE DbgState;
10453 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
10454 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10455
10456 /*
10457 * The loop.
10458 */
10459 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
10460 for (uint32_t cLoops = 0; ; cLoops++)
10461 {
10462 Assert(!HMR0SuspendPending());
10463 HMVMX_ASSERT_CPU_SAFE();
10464 bool fStepping = pVCpu->hm.s.fSingleInstruction;
10465
10466 /*
10467 * Preparatory work for running guest code, this may force us to return
10468 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
10469 */
10470 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
10471 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
10472 rcStrict = hmR0VmxPreRunGuest(pVM, pVCpu, pCtx, &VmxTransient, fStepping);
10473 if (rcStrict != VINF_SUCCESS)
10474 break;
10475
10476 hmR0VmxPreRunGuestCommitted(pVM, pVCpu, pCtx, &VmxTransient);
10477 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
10478
10479 /*
10480 * Now we can run the guest code.
10481 */
10482 int rcRun = hmR0VmxRunGuest(pVM, pVCpu, pCtx);
10483
10484 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
10485
10486 /*
10487 * Restore any residual host-state and save any bits shared between host
10488 * and guest into the guest-CPU state. Re-enables interrupts!
10489 */
10490 hmR0VmxPostRunGuest(pVM, pVCpu, pCtx, &VmxTransient, rcRun);
10491
10492 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
10493 if (RT_SUCCESS(rcRun))
10494 { /* very likely */ }
10495 else
10496 {
10497 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit1, x);
10498 hmR0VmxReportWorldSwitchError(pVM, pVCpu, rcRun, pCtx, &VmxTransient);
10499 return rcRun;
10500 }
10501
10502 /* Profile the VM-exit. */
10503 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
10504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
10505 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
10506 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatExit1, &pVCpu->hm.s.StatExit2, x);
10507 HMVMX_START_EXIT_DISPATCH_PROF();
10508
10509 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
10510
10511 /*
10512 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
10513 */
10514 rcStrict = hmR0VmxRunDebugHandleExit(pVM, pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
10515 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExit2, x);
10516 if (rcStrict != VINF_SUCCESS)
10517 break;
10518 if (cLoops > pVM->hm.s.cMaxResumeLoops)
10519 {
10520 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
10521 rcStrict = VINF_EM_RAW_INTERRUPT;
10522 break;
10523 }
10524
10525 /*
10526 * Stepping: Did the RIP change, if so, consider it a single step.
10527 * Otherwise, make sure one of the TFs gets set.
10528 */
10529 if (fStepping)
10530 {
10531 int rc2 = hmR0VmxSaveGuestRip(pVCpu, pCtx);
10532 rc2 |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pCtx);
10533 AssertRCReturn(rc2, rc2);
10534 if ( pCtx->rip != DbgState.uRipStart
10535 || pCtx->cs.Sel != DbgState.uCsStart)
10536 {
10537 rcStrict = VINF_EM_DBG_STEPPED;
10538 break;
10539 }
10540 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
10541 }
10542
10543 /*
10544 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
10545 */
10546 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
10547 hmR0VmxPreRunGuestDebugStateUpdate(pVM, pVCpu, pCtx, &DbgState, &VmxTransient);
10548 }
10549
10550 /*
10551 * Clear the X86_EFL_TF if necessary.
10552 */
10553 if (pVCpu->hm.s.fClearTrapFlag)
10554 {
10555 int rc2 = hmR0VmxSaveGuestRflags(pVCpu, pCtx);
10556 AssertRCReturn(rc2, rc2);
10557 pVCpu->hm.s.fClearTrapFlag = false;
10558 pCtx->eflags.Bits.u1TF = 0;
10559 }
10560 /** @todo there seems to be issues with the resume flag when the monitor trap
10561 * flag is pending without being used. Seen early in bios init when
10562 * accessing APIC page in protected mode. */
10563
10564 /*
10565 * Restore VM-exit control settings as we may not reenter this function the
10566 * next time around.
10567 */
10568 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10569
10570 /* Restore HMCPU indicators. */
10571 pVCpu->hm.s.fUsingDebugLoop = false;
10572 pVCpu->hm.s.fDebugWantRdTscExit = false;
10573 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10574
10575 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10576 return rcStrict;
10577}
10578
10579
10580/** @} */
10581
10582
10583/**
10584 * Checks if any expensive dtrace probes are enabled and we should go to the
10585 * debug loop.
10586 *
10587 * @returns true if we should use debug loop, false if not.
10588 */
10589static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10590{
10591 /* It's probably faster to OR the raw 32-bit counter variables together.
10592 Since the variables are in an array and the probes are next to one
10593 another (more or less), we have good locality. So, better read
10594 eight-nine cache lines ever time and only have one conditional, than
10595 128+ conditionals, right? */
10596 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10597 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10598 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10599 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10600 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10601 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10602 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10603 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10604 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10605 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10606 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10607 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10608 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10609 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10610 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10611 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10612 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10613 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10614 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10615 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10616 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10617 ) != 0
10618 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10619 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10620 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10621 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10622 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10623 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10624 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10625 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10626 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10627 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10628 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10629 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10630 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10631 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10632 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10633 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10634 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10635 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10636 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10637 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10638 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10639 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10640 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10641 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10642 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10643 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10644 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10645 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10646 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10647 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10648 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10649 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10650 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10651 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10652 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10653 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10654 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10655 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10656 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10657 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10658 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10659 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10660 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10661 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10662 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10663 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10664 ) != 0
10665 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10666 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10667 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10668 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10669 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10670 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10671 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10672 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10673 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10674 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10675 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10676 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10677 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10678 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10679 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10680 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10681 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10682 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10683 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10684 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10685 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10686 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10687 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10688 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10689 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10690 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10691 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10692 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10693 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10694 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10695 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10696 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10697 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10698 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10699 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10700 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10701 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10702 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10703 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10704 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10705 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10706 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10707 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10708 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10709 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10710 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10711 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10712 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10713 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10714 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10715 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10716 ) != 0;
10717}
10718
10719
10720/**
10721 * Runs the guest code using VT-x.
10722 *
10723 * @returns Strict VBox status code (i.e. informational status codes too).
10724 * @param pVM The cross context VM structure.
10725 * @param pVCpu The cross context virtual CPU structure.
10726 * @param pCtx Pointer to the guest-CPU context.
10727 */
10728VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10729{
10730 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10731 Assert(HMVMXCPU_GST_VALUE(pVCpu) == HMVMX_UPDATED_GUEST_ALL);
10732 HMVMX_ASSERT_PREEMPT_SAFE();
10733
10734 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10735
10736 VBOXSTRICTRC rcStrict;
10737 if ( !pVCpu->hm.s.fUseDebugLoop
10738 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10739 && !DBGFIsStepping(pVCpu)
10740 && !pVM->dbgf.ro.cEnabledInt3Breakpoints)
10741 rcStrict = hmR0VmxRunGuestCodeNormal(pVM, pVCpu, pCtx);
10742 else
10743 rcStrict = hmR0VmxRunGuestCodeDebug(pVM, pVCpu, pCtx);
10744
10745 if (rcStrict == VERR_EM_INTERPRETER)
10746 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10747 else if (rcStrict == VINF_EM_RESET)
10748 rcStrict = VINF_EM_TRIPLE_FAULT;
10749
10750 int rc2 = hmR0VmxExitToRing3(pVM, pVCpu, pCtx, rcStrict);
10751 if (RT_FAILURE(rc2))
10752 {
10753 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10754 rcStrict = rc2;
10755 }
10756 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10757 return rcStrict;
10758}
10759
10760
10761#ifndef HMVMX_USE_FUNCTION_TABLE
10762DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10763{
10764# ifdef DEBUG_ramshankar
10765# define RETURN_EXIT_CALL(a_CallExpr) \
10766 do { \
10767 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx); AssertRC(rc2); \
10768 VBOXSTRICTRC rcStrict = a_CallExpr; \
10769 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST); \
10770 return rcStrict; \
10771 } while (0)
10772# else
10773# define RETURN_EXIT_CALL(a_CallExpr) return a_CallExpr
10774# endif
10775 switch (rcReason)
10776 {
10777 case VMX_EXIT_EPT_MISCONFIG: RETURN_EXIT_CALL(hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10778 case VMX_EXIT_EPT_VIOLATION: RETURN_EXIT_CALL(hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10779 case VMX_EXIT_IO_INSTR: RETURN_EXIT_CALL(hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10780 case VMX_EXIT_CPUID: RETURN_EXIT_CALL(hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10781 case VMX_EXIT_RDTSC: RETURN_EXIT_CALL(hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10782 case VMX_EXIT_RDTSCP: RETURN_EXIT_CALL(hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10783 case VMX_EXIT_APIC_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10784 case VMX_EXIT_XCPT_OR_NMI: RETURN_EXIT_CALL(hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10785 case VMX_EXIT_MOV_CRX: RETURN_EXIT_CALL(hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10786 case VMX_EXIT_EXT_INT: RETURN_EXIT_CALL(hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10787 case VMX_EXIT_INT_WINDOW: RETURN_EXIT_CALL(hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10788 case VMX_EXIT_MWAIT: RETURN_EXIT_CALL(hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10789 case VMX_EXIT_MONITOR: RETURN_EXIT_CALL(hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10790 case VMX_EXIT_TASK_SWITCH: RETURN_EXIT_CALL(hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10791 case VMX_EXIT_PREEMPT_TIMER: RETURN_EXIT_CALL(hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10792 case VMX_EXIT_RDMSR: RETURN_EXIT_CALL(hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10793 case VMX_EXIT_WRMSR: RETURN_EXIT_CALL(hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10794 case VMX_EXIT_MOV_DRX: RETURN_EXIT_CALL(hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10795 case VMX_EXIT_TPR_BELOW_THRESHOLD: RETURN_EXIT_CALL(hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10796 case VMX_EXIT_HLT: RETURN_EXIT_CALL(hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10797 case VMX_EXIT_INVD: RETURN_EXIT_CALL(hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10798 case VMX_EXIT_INVLPG: RETURN_EXIT_CALL(hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10799 case VMX_EXIT_RSM: RETURN_EXIT_CALL(hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10800 case VMX_EXIT_MTF: RETURN_EXIT_CALL(hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10801 case VMX_EXIT_PAUSE: RETURN_EXIT_CALL(hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10802 case VMX_EXIT_XDTR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10803 case VMX_EXIT_TR_ACCESS: RETURN_EXIT_CALL(hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10804 case VMX_EXIT_WBINVD: RETURN_EXIT_CALL(hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10805 case VMX_EXIT_XSETBV: RETURN_EXIT_CALL(hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10806 case VMX_EXIT_RDRAND: RETURN_EXIT_CALL(hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10807 case VMX_EXIT_INVPCID: RETURN_EXIT_CALL(hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10808 case VMX_EXIT_GETSEC: RETURN_EXIT_CALL(hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10809 case VMX_EXIT_RDPMC: RETURN_EXIT_CALL(hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10810 case VMX_EXIT_VMCALL: RETURN_EXIT_CALL(hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10811
10812 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10813 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10814 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10815 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10816 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10817 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10818 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10819 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10820 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10821
10822 case VMX_EXIT_VMCLEAR:
10823 case VMX_EXIT_VMLAUNCH:
10824 case VMX_EXIT_VMPTRLD:
10825 case VMX_EXIT_VMPTRST:
10826 case VMX_EXIT_VMREAD:
10827 case VMX_EXIT_VMRESUME:
10828 case VMX_EXIT_VMWRITE:
10829 case VMX_EXIT_VMXOFF:
10830 case VMX_EXIT_VMXON:
10831 case VMX_EXIT_INVEPT:
10832 case VMX_EXIT_INVVPID:
10833 case VMX_EXIT_VMFUNC:
10834 case VMX_EXIT_XSAVES:
10835 case VMX_EXIT_XRSTORS:
10836 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10837 case VMX_EXIT_ENCLS:
10838 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10839 case VMX_EXIT_PML_FULL:
10840 default:
10841 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10842 }
10843#undef RETURN_EXIT_CALL
10844}
10845#endif /* !HMVMX_USE_FUNCTION_TABLE */
10846
10847
10848#ifdef VBOX_STRICT
10849/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10850# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10851 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10852
10853# define HMVMX_ASSERT_PREEMPT_CPUID() \
10854 do { \
10855 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10856 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10857 } while (0)
10858
10859# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10860 do { \
10861 AssertPtr(pVCpu); \
10862 AssertPtr(pMixedCtx); \
10863 AssertPtr(pVmxTransient); \
10864 Assert(pVmxTransient->fVMEntryFailed == false); \
10865 Assert(ASMIntAreEnabled()); \
10866 HMVMX_ASSERT_PREEMPT_SAFE(); \
10867 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10868 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10869 HMVMX_ASSERT_PREEMPT_SAFE(); \
10870 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10871 HMVMX_ASSERT_PREEMPT_CPUID(); \
10872 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10873 } while (0)
10874
10875# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10876 do { \
10877 Log4Func(("\n")); \
10878 } while (0)
10879#else /* nonstrict builds: */
10880# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10881 do { \
10882 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10883 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10884 } while (0)
10885# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10886#endif
10887
10888
10889/**
10890 * Advances the guest RIP by the specified number of bytes.
10891 *
10892 * @param pVCpu The cross context virtual CPU structure.
10893 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10894 * out-of-sync. Make sure to update the required fields
10895 * before using them.
10896 * @param cbInstr Number of bytes to advance the RIP by.
10897 *
10898 * @remarks No-long-jump zone!!!
10899 */
10900DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10901{
10902 /* Advance the RIP. */
10903 pMixedCtx->rip += cbInstr;
10904 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
10905
10906 /* Update interrupt inhibition. */
10907 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10908 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10910}
10911
10912
10913/**
10914 * Advances the guest RIP after reading it from the VMCS.
10915 *
10916 * @returns VBox status code, no informational status codes.
10917 * @param pVCpu The cross context virtual CPU structure.
10918 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10919 * out-of-sync. Make sure to update the required fields
10920 * before using them.
10921 * @param pVmxTransient Pointer to the VMX transient structure.
10922 *
10923 * @remarks No-long-jump zone!!!
10924 */
10925static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10926{
10927 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10928 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
10929 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
10930 AssertRCReturn(rc, rc);
10931
10932 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10933
10934 /*
10935 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10936 * pending debug exception field as it takes care of priority of events.
10937 *
10938 * See Intel spec. 32.2.1 "Debug Exceptions".
10939 */
10940 if ( !pVCpu->hm.s.fSingleInstruction
10941 && pMixedCtx->eflags.Bits.u1TF)
10942 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10943
10944 return VINF_SUCCESS;
10945}
10946
10947
10948/**
10949 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10950 * and update error record fields accordingly.
10951 *
10952 * @return VMX_IGS_* return codes.
10953 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10954 * wrong with the guest state.
10955 *
10956 * @param pVM The cross context VM structure.
10957 * @param pVCpu The cross context virtual CPU structure.
10958 * @param pCtx Pointer to the guest-CPU state.
10959 *
10960 * @remarks This function assumes our cache of the VMCS controls
10961 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10962 */
10963static uint32_t hmR0VmxCheckGuestState(PVM pVM, PVMCPU pVCpu, PCPUMCTX pCtx)
10964{
10965#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10966#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10967 uError = (err); \
10968 break; \
10969 } else do { } while (0)
10970
10971 int rc;
10972 uint32_t uError = VMX_IGS_ERROR;
10973 uint32_t u32Val;
10974 bool fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10975
10976 do
10977 {
10978 /*
10979 * CR0.
10980 */
10981 uint32_t uSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10982 uint32_t uZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10983 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10984 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10985 if (fUnrestrictedGuest)
10986 uSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10987
10988 uint32_t u32GuestCR0;
10989 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCR0);
10990 AssertRCBreak(rc);
10991 HMVMX_CHECK_BREAK((u32GuestCR0 & uSetCR0) == uSetCR0, VMX_IGS_CR0_FIXED1);
10992 HMVMX_CHECK_BREAK(!(u32GuestCR0 & ~uZapCR0), VMX_IGS_CR0_FIXED0);
10993 if ( !fUnrestrictedGuest
10994 && (u32GuestCR0 & X86_CR0_PG)
10995 && !(u32GuestCR0 & X86_CR0_PE))
10996 {
10997 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10998 }
10999
11000 /*
11001 * CR4.
11002 */
11003 uint64_t uSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11004 uint64_t uZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
11005
11006 uint32_t u32GuestCR4;
11007 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCR4);
11008 AssertRCBreak(rc);
11009 HMVMX_CHECK_BREAK((u32GuestCR4 & uSetCR4) == uSetCR4, VMX_IGS_CR4_FIXED1);
11010 HMVMX_CHECK_BREAK(!(u32GuestCR4 & ~uZapCR4), VMX_IGS_CR4_FIXED0);
11011
11012 /*
11013 * IA32_DEBUGCTL MSR.
11014 */
11015 uint64_t u64Val;
11016 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
11017 AssertRCBreak(rc);
11018 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11019 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
11020 {
11021 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
11022 }
11023 uint64_t u64DebugCtlMsr = u64Val;
11024
11025#ifdef VBOX_STRICT
11026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
11027 AssertRCBreak(rc);
11028 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
11029#endif
11030 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
11031
11032 /*
11033 * RIP and RFLAGS.
11034 */
11035 uint32_t u32Eflags;
11036#if HC_ARCH_BITS == 64
11037 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
11038 AssertRCBreak(rc);
11039 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
11040 if ( !fLongModeGuest
11041 || !pCtx->cs.Attr.n.u1Long)
11042 {
11043 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
11044 }
11045 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
11046 * must be identical if the "IA-32e mode guest" VM-entry
11047 * control is 1 and CS.L is 1. No check applies if the
11048 * CPU supports 64 linear-address bits. */
11049
11050 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
11051 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
11052 AssertRCBreak(rc);
11053 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
11054 VMX_IGS_RFLAGS_RESERVED);
11055 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11056 u32Eflags = u64Val;
11057#else
11058 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
11059 AssertRCBreak(rc);
11060 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
11061 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
11062#endif
11063
11064 if ( fLongModeGuest
11065 || ( fUnrestrictedGuest
11066 && !(u32GuestCR0 & X86_CR0_PE)))
11067 {
11068 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
11069 }
11070
11071 uint32_t u32EntryInfo;
11072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
11073 AssertRCBreak(rc);
11074 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11075 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11076 {
11077 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
11078 }
11079
11080 /*
11081 * 64-bit checks.
11082 */
11083#if HC_ARCH_BITS == 64
11084 if (fLongModeGuest)
11085 {
11086 HMVMX_CHECK_BREAK(u32GuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
11087 HMVMX_CHECK_BREAK(u32GuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
11088 }
11089
11090 if ( !fLongModeGuest
11091 && (u32GuestCR4 & X86_CR4_PCIDE))
11092 {
11093 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
11094 }
11095
11096 /** @todo CR3 field must be such that bits 63:52 and bits in the range
11097 * 51:32 beyond the processor's physical-address width are 0. */
11098
11099 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
11100 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
11101 {
11102 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
11103 }
11104
11105 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
11106 AssertRCBreak(rc);
11107 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
11108
11109 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
11110 AssertRCBreak(rc);
11111 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
11112#endif
11113
11114 /*
11115 * PERF_GLOBAL MSR.
11116 */
11117 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
11118 {
11119 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
11120 AssertRCBreak(rc);
11121 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
11122 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
11123 }
11124
11125 /*
11126 * PAT MSR.
11127 */
11128 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
11129 {
11130 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
11131 AssertRCBreak(rc);
11132 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
11133 for (unsigned i = 0; i < 8; i++)
11134 {
11135 uint8_t u8Val = (u64Val & 0xff);
11136 if ( u8Val != 0 /* UC */
11137 && u8Val != 1 /* WC */
11138 && u8Val != 4 /* WT */
11139 && u8Val != 5 /* WP */
11140 && u8Val != 6 /* WB */
11141 && u8Val != 7 /* UC- */)
11142 {
11143 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
11144 }
11145 u64Val >>= 8;
11146 }
11147 }
11148
11149 /*
11150 * EFER MSR.
11151 */
11152 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
11153 {
11154 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
11155 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
11156 AssertRCBreak(rc);
11157 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
11158 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
11159 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
11160 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
11161 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
11162 HMVMX_CHECK_BREAK( fUnrestrictedGuest
11163 || !(u32GuestCR0 & X86_CR0_PG)
11164 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
11165 VMX_IGS_EFER_LMA_LME_MISMATCH);
11166 }
11167
11168 /*
11169 * Segment registers.
11170 */
11171 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11172 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
11173 if (!(u32Eflags & X86_EFL_VM))
11174 {
11175 /* CS */
11176 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
11177 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
11178 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
11179 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
11180 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11181 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
11182 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
11183 /* CS cannot be loaded with NULL in protected mode. */
11184 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
11185 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
11186 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
11187 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
11188 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
11189 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
11190 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
11191 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
11192 else
11193 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
11194
11195 /* SS */
11196 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11197 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
11198 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
11199 if ( !(pCtx->cr0 & X86_CR0_PE)
11200 || pCtx->cs.Attr.n.u4Type == 3)
11201 {
11202 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
11203 }
11204 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
11205 {
11206 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
11207 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
11208 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
11209 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
11210 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
11211 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11212 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
11213 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
11214 }
11215
11216 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
11217 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
11218 {
11219 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
11220 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
11221 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11222 || pCtx->ds.Attr.n.u4Type > 11
11223 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11224 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
11225 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
11226 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
11227 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11228 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
11229 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
11230 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11231 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
11232 }
11233 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
11234 {
11235 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
11236 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
11237 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11238 || pCtx->es.Attr.n.u4Type > 11
11239 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
11240 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
11241 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
11242 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
11243 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11244 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
11245 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
11246 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11247 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
11248 }
11249 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
11250 {
11251 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
11252 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
11253 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11254 || pCtx->fs.Attr.n.u4Type > 11
11255 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
11256 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
11257 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
11258 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
11259 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11260 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
11261 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
11262 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11263 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
11264 }
11265 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
11266 {
11267 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
11268 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
11269 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
11270 || pCtx->gs.Attr.n.u4Type > 11
11271 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
11272 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
11273 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
11274 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
11275 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11276 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
11277 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
11278 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
11279 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
11280 }
11281 /* 64-bit capable CPUs. */
11282#if HC_ARCH_BITS == 64
11283 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11284 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11285 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11286 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11287 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11288 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11289 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11290 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11291 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11292 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11293 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11294#endif
11295 }
11296 else
11297 {
11298 /* V86 mode checks. */
11299 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
11300 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11301 {
11302 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
11303 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
11304 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
11305 }
11306 else
11307 {
11308 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
11309 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
11310 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
11311 }
11312
11313 /* CS */
11314 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
11315 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
11316 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
11317 /* SS */
11318 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
11319 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
11320 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
11321 /* DS */
11322 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
11323 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
11324 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
11325 /* ES */
11326 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
11327 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
11328 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
11329 /* FS */
11330 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
11331 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
11332 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
11333 /* GS */
11334 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
11335 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
11336 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
11337 /* 64-bit capable CPUs. */
11338#if HC_ARCH_BITS == 64
11339 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
11340 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
11341 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
11342 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
11343 HMVMX_CHECK_BREAK(!(pCtx->cs.u64Base >> 32), VMX_IGS_LONGMODE_CS_BASE_INVALID);
11344 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ss.u64Base >> 32),
11345 VMX_IGS_LONGMODE_SS_BASE_INVALID);
11346 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->ds.u64Base >> 32),
11347 VMX_IGS_LONGMODE_DS_BASE_INVALID);
11348 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !(pCtx->es.u64Base >> 32),
11349 VMX_IGS_LONGMODE_ES_BASE_INVALID);
11350#endif
11351 }
11352
11353 /*
11354 * TR.
11355 */
11356 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
11357 /* 64-bit capable CPUs. */
11358#if HC_ARCH_BITS == 64
11359 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
11360#endif
11361 if (fLongModeGuest)
11362 {
11363 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
11364 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
11365 }
11366 else
11367 {
11368 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
11369 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
11370 VMX_IGS_TR_ATTR_TYPE_INVALID);
11371 }
11372 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
11373 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
11374 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
11375 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
11376 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11377 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
11378 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
11379 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
11380
11381 /*
11382 * GDTR and IDTR.
11383 */
11384#if HC_ARCH_BITS == 64
11385 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
11386 AssertRCBreak(rc);
11387 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
11388
11389 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
11390 AssertRCBreak(rc);
11391 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
11392#endif
11393
11394 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
11395 AssertRCBreak(rc);
11396 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11397
11398 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
11399 AssertRCBreak(rc);
11400 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
11401
11402 /*
11403 * Guest Non-Register State.
11404 */
11405 /* Activity State. */
11406 uint32_t u32ActivityState;
11407 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
11408 AssertRCBreak(rc);
11409 HMVMX_CHECK_BREAK( !u32ActivityState
11410 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
11411 VMX_IGS_ACTIVITY_STATE_INVALID);
11412 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
11413 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
11414 uint32_t u32IntrState;
11415 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
11416 AssertRCBreak(rc);
11417 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
11418 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11419 {
11420 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
11421 }
11422
11423 /** @todo Activity state and injecting interrupts. Left as a todo since we
11424 * currently don't use activity states but ACTIVE. */
11425
11426 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11427 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
11428
11429 /* Guest interruptibility-state. */
11430 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
11431 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11432 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
11433 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
11434 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11435 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
11436 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
11437 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11438 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
11439 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
11440 {
11441 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
11442 {
11443 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11444 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11445 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
11446 }
11447 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11448 {
11449 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
11450 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
11451 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
11452 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
11453 }
11454 }
11455 /** @todo Assumes the processor is not in SMM. */
11456 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11457 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
11458 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
11459 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
11460 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
11461 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
11462 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
11463 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11464 {
11465 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
11466 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
11467 }
11468
11469 /* Pending debug exceptions. */
11470#if HC_ARCH_BITS == 64
11471 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
11472 AssertRCBreak(rc);
11473 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
11474 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
11475 u32Val = u64Val; /* For pending debug exceptions checks below. */
11476#else
11477 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
11478 AssertRCBreak(rc);
11479 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
11480 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
11481#endif
11482
11483 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
11484 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
11485 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
11486 {
11487 if ( (u32Eflags & X86_EFL_TF)
11488 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11489 {
11490 /* Bit 14 is PendingDebug.BS. */
11491 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
11492 }
11493 if ( !(u32Eflags & X86_EFL_TF)
11494 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
11495 {
11496 /* Bit 14 is PendingDebug.BS. */
11497 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
11498 }
11499 }
11500
11501 /* VMCS link pointer. */
11502 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
11503 AssertRCBreak(rc);
11504 if (u64Val != UINT64_C(0xffffffffffffffff))
11505 {
11506 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
11507 /** @todo Bits beyond the processor's physical-address width MBZ. */
11508 /** @todo 32-bit located in memory referenced by value of this field (as a
11509 * physical address) must contain the processor's VMCS revision ID. */
11510 /** @todo SMM checks. */
11511 }
11512
11513 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
11514 * not using Nested Paging? */
11515 if ( pVM->hm.s.fNestedPaging
11516 && !fLongModeGuest
11517 && CPUMIsGuestInPAEModeEx(pCtx))
11518 {
11519 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
11520 AssertRCBreak(rc);
11521 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11522
11523 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
11524 AssertRCBreak(rc);
11525 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11526
11527 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
11528 AssertRCBreak(rc);
11529 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11530
11531 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
11532 AssertRCBreak(rc);
11533 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
11534 }
11535
11536 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
11537 if (uError == VMX_IGS_ERROR)
11538 uError = VMX_IGS_REASON_NOT_FOUND;
11539 } while (0);
11540
11541 pVCpu->hm.s.u32HMError = uError;
11542 return uError;
11543
11544#undef HMVMX_ERROR_BREAK
11545#undef HMVMX_CHECK_BREAK
11546}
11547
11548/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11549/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11550/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11551
11552/** @name VM-exit handlers.
11553 * @{
11554 */
11555
11556/**
11557 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11558 */
11559HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11560{
11561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11563 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11564 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11565 return VINF_SUCCESS;
11566 return VINF_EM_RAW_INTERRUPT;
11567}
11568
11569
11570/**
11571 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11572 */
11573HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11574{
11575 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11576 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11577
11578 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11579 AssertRCReturn(rc, rc);
11580
11581 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11582 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11583 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11584 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11585
11586 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11587 {
11588 /*
11589 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we injected it ourselves and
11590 * anything we inject is not going to cause a VM-exit directly for the event being injected.
11591 * See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11592 *
11593 * Dispatch the NMI to the host. See Intel spec. 27.5.5 "Updating Non-Register State".
11594 */
11595 VMXDispatchHostNmi();
11596 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11597 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11598 return VINF_SUCCESS;
11599 }
11600
11601 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11602 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11603 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11604 { /* likely */ }
11605 else
11606 {
11607 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11608 rcStrictRc1 = VINF_SUCCESS;
11609 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11610 return rcStrictRc1;
11611 }
11612
11613 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11614 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11615 switch (uIntType)
11616 {
11617 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11618 Assert(uVector == X86_XCPT_DB);
11619 RT_FALL_THRU();
11620 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11621 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11622 RT_FALL_THRU();
11623 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11624 {
11625 /*
11626 * If there's any exception caused as a result of event injection, the resulting
11627 * secondary/final execption will be pending, we shall continue guest execution
11628 * after injecting the event. The page-fault case is complicated and we manually
11629 * handle any currently pending event in hmR0VmxExitXcptPF.
11630 */
11631 if (!pVCpu->hm.s.Event.fPending)
11632 { /* likely */ }
11633 else if (uVector != X86_XCPT_PF)
11634 {
11635 rc = VINF_SUCCESS;
11636 break;
11637 }
11638
11639 switch (uVector)
11640 {
11641 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11642 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11643 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11644 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11645 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11646 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11647
11648 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11649 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11650 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11651 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11652 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11653 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11654 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11655 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11656 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11657 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11658 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11659 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11660 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11661 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11662 default:
11663 {
11664 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11665 AssertRCReturn(rc, rc);
11666
11667 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11668 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11669 {
11670 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11671 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11672 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11673
11674 rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11675 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11676 AssertRCReturn(rc, rc);
11677 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11678 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11679 0 /* GCPtrFaultAddress */);
11680 AssertRCReturn(rc, rc);
11681 }
11682 else
11683 {
11684 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11685 pVCpu->hm.s.u32HMError = uVector;
11686 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11687 }
11688 break;
11689 }
11690 }
11691 break;
11692 }
11693
11694 default:
11695 {
11696 pVCpu->hm.s.u32HMError = uExitIntInfo;
11697 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11698 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11699 break;
11700 }
11701 }
11702 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11703 return rc;
11704}
11705
11706
11707/**
11708 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11709 */
11710HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11711{
11712 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11713
11714 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11715 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11716
11717 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11718 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11719 return VINF_SUCCESS;
11720}
11721
11722
11723/**
11724 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11725 */
11726HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11727{
11728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11729 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11730 {
11731 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11732 HMVMX_RETURN_UNEXPECTED_EXIT();
11733 }
11734
11735 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11736
11737 /*
11738 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11739 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11740 */
11741 uint32_t uIntrState = 0;
11742 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
11743 AssertRCReturn(rc, rc);
11744
11745 bool const fBlockSti = RT_BOOL(uIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11746 if ( fBlockSti
11747 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11748 {
11749 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11750 }
11751
11752 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11753 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11754
11755 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11756 return VINF_SUCCESS;
11757}
11758
11759
11760/**
11761 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11762 */
11763HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11766 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWbinvd);
11767 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11768}
11769
11770
11771/**
11772 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11773 */
11774HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11775{
11776 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvd);
11778 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11779}
11780
11781
11782/**
11783 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11784 */
11785HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11786{
11787 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11788 PVM pVM = pVCpu->CTX_SUFF(pVM);
11789 int rc = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11790 if (RT_LIKELY(rc == VINF_SUCCESS))
11791 {
11792 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11793 Assert(pVmxTransient->cbInstr == 2);
11794 }
11795 else
11796 {
11797 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11798 rc = VERR_EM_INTERPRETER;
11799 }
11800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCpuid);
11801 return rc;
11802}
11803
11804
11805/**
11806 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11807 */
11808HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11809{
11810 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11811 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11812 AssertRCReturn(rc, rc);
11813
11814 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11815 return VINF_EM_RAW_EMULATE_INSTR;
11816
11817 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11818 HMVMX_RETURN_UNEXPECTED_EXIT();
11819}
11820
11821
11822/**
11823 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11824 */
11825HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11826{
11827 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11828 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11829 AssertRCReturn(rc, rc);
11830
11831 PVM pVM = pVCpu->CTX_SUFF(pVM);
11832 rc = EMInterpretRdtsc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11833 if (RT_LIKELY(rc == VINF_SUCCESS))
11834 {
11835 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11836 Assert(pVmxTransient->cbInstr == 2);
11837 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11838 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11839 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11840 }
11841 else
11842 rc = VERR_EM_INTERPRETER;
11843 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11844 return rc;
11845}
11846
11847
11848/**
11849 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11850 */
11851HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11852{
11853 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11854 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11855 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx); /* For MSR_K8_TSC_AUX */
11856 AssertRCReturn(rc, rc);
11857
11858 PVM pVM = pVCpu->CTX_SUFF(pVM);
11859 rc = EMInterpretRdtscp(pVM, pVCpu, pMixedCtx);
11860 if (RT_SUCCESS(rc))
11861 {
11862 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11863 Assert(pVmxTransient->cbInstr == 3);
11864 /* If we get a spurious VM-exit when offsetting is enabled, we must reset offsetting on VM-reentry. See @bugref{6634}. */
11865 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11866 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11867 }
11868 else
11869 {
11870 AssertMsgFailed(("hmR0VmxExitRdtscp: EMInterpretRdtscp failed with %Rrc\n", rc));
11871 rc = VERR_EM_INTERPRETER;
11872 }
11873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdtsc);
11874 return rc;
11875}
11876
11877
11878/**
11879 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11880 */
11881HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11882{
11883 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11884 int rc = hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
11885 rc |= hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11886 AssertRCReturn(rc, rc);
11887
11888 PVM pVM = pVCpu->CTX_SUFF(pVM);
11889 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11890 if (RT_LIKELY(rc == VINF_SUCCESS))
11891 {
11892 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11893 Assert(pVmxTransient->cbInstr == 2);
11894 }
11895 else
11896 {
11897 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11898 rc = VERR_EM_INTERPRETER;
11899 }
11900 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdpmc);
11901 return rc;
11902}
11903
11904
11905/**
11906 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11907 */
11908HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11909{
11910 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitVmcall);
11912
11913 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11914 if (EMAreHypercallInstructionsEnabled(pVCpu))
11915 {
11916#if 0
11917 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
11918#else
11919 /* Aggressive state sync. for now. */
11920 int rc = hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
11921 rc |= hmR0VmxSaveGuestRflags(pVCpu,pMixedCtx); /* For CPL checks in gimHvHypercall() & gimKvmHypercall() */
11922 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* For long-mode checks in gimKvmHypercall(). */
11923 AssertRCReturn(rc, rc);
11924#endif
11925
11926 /* Perform the hypercall. */
11927 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11928 if (rcStrict == VINF_SUCCESS)
11929 {
11930 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11931 AssertRCReturn(rc, rc);
11932 }
11933 else
11934 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11935 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11936 || RT_FAILURE(rcStrict));
11937
11938 /* If the hypercall changes anything other than guest's general-purpose registers,
11939 we would need to reload the guest changed bits here before VM-entry. */
11940 }
11941 else
11942 Log4(("hmR0VmxExitVmcall: Hypercalls not enabled\n"));
11943
11944 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11945 if (RT_FAILURE(rcStrict))
11946 {
11947 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11948 rcStrict = VINF_SUCCESS;
11949 }
11950
11951 return rcStrict;
11952}
11953
11954
11955/**
11956 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11957 */
11958HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11959{
11960 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11961 PVM pVM = pVCpu->CTX_SUFF(pVM);
11962 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11963
11964 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11965 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
11966 AssertRCReturn(rc, rc);
11967
11968 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11969 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11970 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11971 else
11972 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11973 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInvlpg);
11975 return rcStrict;
11976}
11977
11978
11979/**
11980 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11981 */
11982HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11983{
11984 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11985 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
11986 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
11987 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
11988 AssertRCReturn(rc, rc);
11989
11990 PVM pVM = pVCpu->CTX_SUFF(pVM);
11991 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11992 if (RT_LIKELY(rc == VINF_SUCCESS))
11993 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11994 else
11995 {
11996 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11997 rc = VERR_EM_INTERPRETER;
11998 }
11999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
12000 return rc;
12001}
12002
12003
12004/**
12005 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
12006 */
12007HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12008{
12009 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12010 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12011 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12012 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12013 AssertRCReturn(rc, rc);
12014
12015 PVM pVM = pVCpu->CTX_SUFF(pVM);
12016 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12017 rc = VBOXSTRICTRC_VAL(rc2);
12018 if (RT_LIKELY( rc == VINF_SUCCESS
12019 || rc == VINF_EM_HALT))
12020 {
12021 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12022 AssertRCReturn(rc3, rc3);
12023
12024 if ( rc == VINF_EM_HALT
12025 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
12026 {
12027 rc = VINF_SUCCESS;
12028 }
12029 }
12030 else
12031 {
12032 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
12033 rc = VERR_EM_INTERPRETER;
12034 }
12035 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
12036 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
12037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
12038 return rc;
12039}
12040
12041
12042/**
12043 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
12044 */
12045HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12046{
12047 /*
12048 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root mode. In theory, we should never
12049 * get this VM-exit. This can happen only if dual-monitor treatment of SMI and VMX is enabled, which can (only?) be done by
12050 * executing VMCALL in VMX root operation. If we get here, something funny is going on.
12051 * See Intel spec. "33.15.5 Enabling the Dual-Monitor Treatment".
12052 */
12053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12054 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12055 HMVMX_RETURN_UNEXPECTED_EXIT();
12056}
12057
12058
12059/**
12060 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
12061 */
12062HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12063{
12064 /*
12065 * This can only happen if we support dual-monitor treatment of SMI, which can be activated by executing VMCALL in VMX
12066 * root operation. Only an STM (SMM transfer monitor) would get this VM-exit when we (the executive monitor) execute a VMCALL
12067 * in VMX root mode or receive an SMI. If we get here, something funny is going on.
12068 * See Intel spec. "33.15.6 Activating the Dual-Monitor Treatment" and Intel spec. 25.3 "Other Causes of VM-Exits"
12069 */
12070 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12071 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12072 HMVMX_RETURN_UNEXPECTED_EXIT();
12073}
12074
12075
12076/**
12077 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
12078 */
12079HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12080{
12081 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
12082 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12083 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12084 HMVMX_RETURN_UNEXPECTED_EXIT();
12085}
12086
12087
12088/**
12089 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
12090 */
12091HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12092{
12093 /*
12094 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used. We currently
12095 * don't make use of it (see hmR0VmxLoadGuestActivityState()) as our guests don't have direct access to the host LAPIC.
12096 * See Intel spec. 25.3 "Other Causes of VM-exits".
12097 */
12098 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12099 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12100 HMVMX_RETURN_UNEXPECTED_EXIT();
12101}
12102
12103
12104/**
12105 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
12106 * VM-exit.
12107 */
12108HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12109{
12110 /*
12111 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
12112 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
12113 *
12114 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
12115 * See Intel spec. "23.8 Restrictions on VMX operation".
12116 */
12117 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12118 return VINF_SUCCESS;
12119}
12120
12121
12122/**
12123 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
12124 * VM-exit.
12125 */
12126HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12127{
12128 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12129 return VINF_EM_RESET;
12130}
12131
12132
12133/**
12134 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
12135 */
12136HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12137{
12138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12139 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
12140
12141 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12142 AssertRCReturn(rc, rc);
12143
12144 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
12145 rc = VINF_SUCCESS;
12146 else
12147 rc = VINF_EM_HALT;
12148
12149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12150 if (rc != VINF_SUCCESS)
12151 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
12152 return rc;
12153}
12154
12155
12156/**
12157 * VM-exit handler for instructions that result in a \#UD exception delivered to
12158 * the guest.
12159 */
12160HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12161{
12162 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12163 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
12164 return VINF_SUCCESS;
12165}
12166
12167
12168/**
12169 * VM-exit handler for expiry of the VMX preemption timer.
12170 */
12171HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12172{
12173 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12174
12175 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
12176 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12177
12178 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
12179 PVM pVM = pVCpu->CTX_SUFF(pVM);
12180 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
12181 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
12182 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
12183}
12184
12185
12186/**
12187 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
12188 */
12189HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12190{
12191 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12192
12193 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12194 rc |= hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, false /*fNeedRsp*/);
12195 rc |= hmR0VmxSaveGuestCR4(pVCpu, pMixedCtx);
12196 AssertRCReturn(rc, rc);
12197
12198 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
12199 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12200
12201 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
12202
12203 return rcStrict;
12204}
12205
12206
12207/**
12208 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
12209 */
12210HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12211{
12212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12213 /** @todo Use VM-exit instruction information. */
12214 return VERR_EM_INTERPRETER;
12215}
12216
12217
12218/**
12219 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
12220 * Error VM-exit.
12221 */
12222HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12223{
12224 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12225 AssertRCReturn(rc, rc);
12226
12227 rc = hmR0VmxCheckVmcsCtls(pVCpu);
12228 AssertRCReturn(rc, rc);
12229
12230 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12231 NOREF(uInvalidReason);
12232
12233#ifdef VBOX_STRICT
12234 uint32_t uIntrState;
12235 RTHCUINTREG uHCReg;
12236 uint64_t u64Val;
12237 uint32_t u32Val;
12238
12239 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
12240 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
12241 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
12242 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &uIntrState);
12243 AssertRCReturn(rc, rc);
12244
12245 Log4(("uInvalidReason %u\n", uInvalidReason));
12246 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
12247 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
12248 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
12249 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", uIntrState));
12250
12251 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
12252 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
12253 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
12254 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
12255 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
12256 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12257 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
12258 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
12259 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
12260 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
12261 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
12262 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
12263#else
12264 NOREF(pVmxTransient);
12265#endif
12266
12267 hmR0DumpRegs(pVCpu->CTX_SUFF(pVM), pVCpu, pMixedCtx);
12268 return VERR_VMX_INVALID_GUEST_STATE;
12269}
12270
12271
12272/**
12273 * VM-exit handler for VM-entry failure due to an MSR-load
12274 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
12275 */
12276HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12277{
12278 NOREF(pVmxTransient);
12279 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12280 HMVMX_RETURN_UNEXPECTED_EXIT();
12281}
12282
12283
12284/**
12285 * VM-exit handler for VM-entry failure due to a machine-check event
12286 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
12287 */
12288HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12289{
12290 NOREF(pVmxTransient);
12291 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
12292 HMVMX_RETURN_UNEXPECTED_EXIT();
12293}
12294
12295
12296/**
12297 * VM-exit handler for all undefined reasons. Should never ever happen.. in
12298 * theory.
12299 */
12300HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12301{
12302 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
12303 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
12304 return VERR_VMX_UNDEFINED_EXIT_CODE;
12305}
12306
12307
12308/**
12309 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
12310 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
12311 * Conditional VM-exit.
12312 */
12313HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12314{
12315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12316
12317 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
12318 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
12319 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
12320 return VERR_EM_INTERPRETER;
12321 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12322 HMVMX_RETURN_UNEXPECTED_EXIT();
12323}
12324
12325
12326/**
12327 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
12328 */
12329HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12330{
12331 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12332
12333 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
12334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdrand);
12335 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
12336 return VERR_EM_INTERPRETER;
12337 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12338 HMVMX_RETURN_UNEXPECTED_EXIT();
12339}
12340
12341
12342/**
12343 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
12344 */
12345HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12346{
12347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12348
12349 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
12350 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12351 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12352 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12353 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12354 {
12355 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12356 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12357 }
12358 AssertRCReturn(rc, rc);
12359 Log4(("ecx=%#RX32\n", pMixedCtx->ecx));
12360
12361#ifdef VBOX_STRICT
12362 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
12363 {
12364 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
12365 && pMixedCtx->ecx != MSR_K6_EFER)
12366 {
12367 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12368 pMixedCtx->ecx));
12369 HMVMX_RETURN_UNEXPECTED_EXIT();
12370 }
12371 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12372 {
12373 VMXMSREXITREAD enmRead;
12374 VMXMSREXITWRITE enmWrite;
12375 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12376 AssertRCReturn(rc2, rc2);
12377 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
12378 {
12379 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12380 HMVMX_RETURN_UNEXPECTED_EXIT();
12381 }
12382 }
12383 }
12384#endif
12385
12386 PVM pVM = pVCpu->CTX_SUFF(pVM);
12387 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12388 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
12389 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
12390 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
12391 if (RT_SUCCESS(rc))
12392 {
12393 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12394 Assert(pVmxTransient->cbInstr == 2);
12395 }
12396 return rc;
12397}
12398
12399
12400/**
12401 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
12402 */
12403HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12404{
12405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12406 PVM pVM = pVCpu->CTX_SUFF(pVM);
12407 int rc = VINF_SUCCESS;
12408
12409 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
12410 rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
12411 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx);
12412 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
12413 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12414 {
12415 rc |= hmR0VmxSaveGuestLazyMsrs(pVCpu, pMixedCtx);
12416 rc |= hmR0VmxSaveGuestAutoLoadStoreMsrs(pVCpu, pMixedCtx);
12417 }
12418 AssertRCReturn(rc, rc);
12419 Log4(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
12420
12421 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
12422 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
12423 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
12424
12425 if (RT_SUCCESS(rc))
12426 {
12427 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12428
12429 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
12430 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
12431 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
12432 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
12433 {
12434 /*
12435 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
12436 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
12437 * EMInterpretWrmsr() changes it.
12438 */
12439 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12440 }
12441 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
12442 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
12443 else if (pMixedCtx->ecx == MSR_K6_EFER)
12444 {
12445 /*
12446 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
12447 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
12448 * the other bits as well, SCE and NXE. See @bugref{7368}.
12449 */
12450 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS | HM_CHANGED_VMX_EXIT_CTLS);
12451 }
12452
12453 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
12454 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
12455 {
12456 switch (pMixedCtx->ecx)
12457 {
12458 /*
12459 * For SYSENTER CS, EIP, ESP MSRs, we set both the flags here so we don't accidentally
12460 * overwrite the changed guest-CPU context value while going to ring-3, see @bufref{8745}.
12461 */
12462 case MSR_IA32_SYSENTER_CS:
12463 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_CS_MSR);
12464 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_CS_MSR);
12465 break;
12466 case MSR_IA32_SYSENTER_EIP:
12467 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
12468 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_EIP_MSR);
12469 break;
12470 case MSR_IA32_SYSENTER_ESP:
12471 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
12472 HMVMXCPU_GST_SET_UPDATED(pVCpu, HMVMX_UPDATED_GUEST_SYSENTER_ESP_MSR);
12473 break;
12474 case MSR_K8_FS_BASE: RT_FALL_THRU();
12475 case MSR_K8_GS_BASE: HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_SEGMENT_REGS); break;
12476 case MSR_K6_EFER: /* already handled above */ break;
12477 default:
12478 {
12479 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12480 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
12481 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12482 HMCPU_CF_SET(pVCpu, HM_CHANGED_VMM_GUEST_LAZY_MSRS);
12483 break;
12484 }
12485 }
12486 }
12487#ifdef VBOX_STRICT
12488 else
12489 {
12490 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
12491 switch (pMixedCtx->ecx)
12492 {
12493 case MSR_IA32_SYSENTER_CS:
12494 case MSR_IA32_SYSENTER_EIP:
12495 case MSR_IA32_SYSENTER_ESP:
12496 case MSR_K8_FS_BASE:
12497 case MSR_K8_GS_BASE:
12498 {
12499 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
12500 HMVMX_RETURN_UNEXPECTED_EXIT();
12501 }
12502
12503 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
12504 default:
12505 {
12506 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
12507 {
12508 /* EFER writes are always intercepted, see hmR0VmxLoadGuestMsrs(). */
12509 if (pMixedCtx->ecx != MSR_K6_EFER)
12510 {
12511 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
12512 pMixedCtx->ecx));
12513 HMVMX_RETURN_UNEXPECTED_EXIT();
12514 }
12515 }
12516
12517 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
12518 {
12519 VMXMSREXITREAD enmRead;
12520 VMXMSREXITWRITE enmWrite;
12521 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
12522 AssertRCReturn(rc2, rc2);
12523 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
12524 {
12525 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
12526 HMVMX_RETURN_UNEXPECTED_EXIT();
12527 }
12528 }
12529 break;
12530 }
12531 }
12532 }
12533#endif /* VBOX_STRICT */
12534 }
12535 return rc;
12536}
12537
12538
12539/**
12540 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12541 */
12542HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12543{
12544 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12545
12546 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPause);
12547 return VINF_EM_RAW_INTERRUPT;
12548}
12549
12550
12551/**
12552 * VM-exit handler for when the TPR value is lowered below the specified
12553 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12554 */
12555HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12556{
12557 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12558 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12559
12560 /*
12561 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12562 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12563 */
12564 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12565 return VINF_SUCCESS;
12566}
12567
12568
12569/**
12570 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12571 * VM-exit.
12572 *
12573 * @retval VINF_SUCCESS when guest execution can continue.
12574 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12575 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12576 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12577 * interpreter.
12578 */
12579HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12580{
12581 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12582 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12583 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12584 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12585 AssertRCReturn(rc, rc);
12586
12587 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12588 uint32_t const uAccessType = VMX_EXIT_QUALIFICATION_CRX_ACCESS(uExitQualification);
12589 PVM pVM = pVCpu->CTX_SUFF(pVM);
12590 VBOXSTRICTRC rcStrict;
12591 rc = hmR0VmxSaveGuestRegsForIemExec(pVCpu, pMixedCtx, false /*fMemory*/, true /*fNeedRsp*/);
12592 switch (uAccessType)
12593 {
12594 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_WRITE: /* MOV to CRx */
12595 {
12596 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12597 AssertRCReturn(rc, rc);
12598
12599 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12600 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12601 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification));
12602 AssertMsg( rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE
12603 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12604 switch (VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification))
12605 {
12606 case 0: /* CR0 */
12607 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12608 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12609 break;
12610 case 2: /* CR2 */
12611 /* Nothing to do here, CR2 it's not part of the VMCS. */
12612 break;
12613 case 3: /* CR3 */
12614 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12615 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR3);
12616 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12617 break;
12618 case 4: /* CR4 */
12619 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR4);
12620 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n",
12621 VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12622 break;
12623 case 8: /* CR8 */
12624 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12625 /* CR8 contains the APIC TPR. Was updated by IEMExecDecodedMovCRxWrite(). */
12626 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_APIC_STATE);
12627 break;
12628 default:
12629 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)));
12630 break;
12631 }
12632
12633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxWrite[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12634 break;
12635 }
12636
12637 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_READ: /* MOV from CRx */
12638 {
12639 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
12640 AssertRCReturn(rc, rc);
12641
12642 Assert( !pVM->hm.s.fNestedPaging
12643 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12644 || pVCpu->hm.s.fUsingDebugLoop
12645 || VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 3);
12646
12647 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12648 Assert( VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification) != 8
12649 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12650
12651 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12652 VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification),
12653 VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification));
12654 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12655 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCRxRead[VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification)]);
12656 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUALIFICATION_CRX_REGISTER(uExitQualification),
12657 VBOXSTRICTRC_VAL(rcStrict)));
12658 if (VMX_EXIT_QUALIFICATION_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12659 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RSP);
12660 break;
12661 }
12662
12663 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12664 {
12665 AssertRCReturn(rc, rc);
12666 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12667 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12668 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12670 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12671 break;
12672 }
12673
12674 case VMX_EXIT_QUALIFICATION_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12675 {
12676 AssertRCReturn(rc, rc);
12677 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12678 VMX_EXIT_QUALIFICATION_CRX_LMSW_DATA(uExitQualification));
12679 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_IEM_RAISED_XCPT || rcStrict == VINF_PGM_CHANGE_MODE,
12680 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12681 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_CR0);
12682 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12683 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12684 break;
12685 }
12686
12687 default:
12688 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12689 VERR_VMX_UNEXPECTED_EXCEPTION);
12690 }
12691
12692 HMCPU_CF_SET(pVCpu, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS : HM_CHANGED_ALL_GUEST);
12693 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12694 NOREF(pVM);
12695 return rcStrict;
12696}
12697
12698
12699/**
12700 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12701 * VM-exit.
12702 */
12703HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12704{
12705 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12706 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12707
12708 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12709 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12710 rc |= hmR0VmxSaveGuestRip(pVCpu, pMixedCtx);
12711 rc |= hmR0VmxSaveGuestRflags(pVCpu, pMixedCtx); /* Eflag checks in EMInterpretDisasCurrent(). */
12712 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx); /* CR0 checks & PGM* in EMInterpretDisasCurrent(). */
12713 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx); /* SELM checks in EMInterpretDisasCurrent(). */
12714 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12715 AssertRCReturn(rc, rc);
12716
12717 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12718 uint32_t uIOPort = VMX_EXIT_QUALIFICATION_IO_PORT(pVmxTransient->uExitQualification);
12719 uint8_t uIOWidth = VMX_EXIT_QUALIFICATION_IO_WIDTH(pVmxTransient->uExitQualification);
12720 bool fIOWrite = ( VMX_EXIT_QUALIFICATION_IO_DIRECTION(pVmxTransient->uExitQualification)
12721 == VMX_EXIT_QUALIFICATION_IO_DIRECTION_OUT);
12722 bool fIOString = VMX_EXIT_QUALIFICATION_IO_IS_STRING(pVmxTransient->uExitQualification);
12723 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12724 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12725 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12726
12727 /* I/O operation lookup arrays. */
12728 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12729 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving the result (in AL/AX/EAX). */
12730
12731 VBOXSTRICTRC rcStrict;
12732 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12733 uint32_t const cbInstr = pVmxTransient->cbInstr;
12734 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12735 PVM pVM = pVCpu->CTX_SUFF(pVM);
12736 if (fIOString)
12737 {
12738#ifdef VBOX_WITH_2ND_IEM_STEP /* This used to gurus with debian 32-bit guest without NP (on ATA reads).
12739 See @bugref{5752#c158}. Should work now. */
12740 /*
12741 * INS/OUTS - I/O String instruction.
12742 *
12743 * Use instruction-information if available, otherwise fall back on
12744 * interpreting the instruction.
12745 */
12746 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12747 fIOWrite ? 'w' : 'r'));
12748 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12749 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12750 {
12751 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12752 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12753 rc2 |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12754 AssertRCReturn(rc2, rc2);
12755 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12756 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12757 IEMMODE enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12758 bool fRep = VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification);
12759 if (fIOWrite)
12760 {
12761 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12762 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12763 }
12764 else
12765 {
12766 /*
12767 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12768 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12769 * See Intel Instruction spec. for "INS".
12770 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12771 */
12772 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12773 }
12774 }
12775 else
12776 {
12777 /** @todo optimize this, IEM should request the additional state if it needs it (GP, PF, ++). */
12778 int rc2 = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
12779 AssertRCReturn(rc2, rc2);
12780 rcStrict = IEMExecOne(pVCpu);
12781 }
12782 /** @todo IEM needs to be setting these flags somehow. */
12783 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12784 fUpdateRipAlready = true;
12785#else
12786 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12787 rcStrict = EMInterpretDisasCurrent(pVM, pVCpu, pDis, NULL /* pcbInstr */);
12788 if (RT_SUCCESS(rcStrict))
12789 {
12790 if (fIOWrite)
12791 {
12792 rcStrict = IOMInterpretOUTSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12793 (DISCPUMODE)pDis->uAddrMode, cbValue);
12794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringWrite);
12795 }
12796 else
12797 {
12798 rcStrict = IOMInterpretINSEx(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), uIOPort, pDis->fPrefix,
12799 (DISCPUMODE)pDis->uAddrMode, cbValue);
12800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOStringRead);
12801 }
12802 }
12803 else
12804 {
12805 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("rcStrict=%Rrc RIP=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict),
12806 pMixedCtx->rip));
12807 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
12808 }
12809#endif
12810 }
12811 else
12812 {
12813 /*
12814 * IN/OUT - I/O instruction.
12815 */
12816 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12817 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12818 Assert(!VMX_EXIT_QUALIFICATION_IO_IS_REP(pVmxTransient->uExitQualification));
12819 if (fIOWrite)
12820 {
12821 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12823 }
12824 else
12825 {
12826 uint32_t u32Result = 0;
12827 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12828 if (IOM_SUCCESS(rcStrict))
12829 {
12830 /* Save result of I/O IN instr. in AL/AX/EAX. */
12831 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12832 }
12833 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12834 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12835 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12836 }
12837 }
12838
12839 if (IOM_SUCCESS(rcStrict))
12840 {
12841 if (!fUpdateRipAlready)
12842 {
12843 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12844 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP);
12845 }
12846
12847 /*
12848 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru while booting Fedora 17 64-bit guest.
12849 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12850 */
12851 if (fIOString)
12852 {
12853 /** @todo Single-step for INS/OUTS with REP prefix? */
12854 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
12855 }
12856 else if ( !fDbgStepping
12857 && fGstStepping)
12858 {
12859 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12860 }
12861
12862 /*
12863 * If any I/O breakpoints are armed, we need to check if one triggered
12864 * and take appropriate action.
12865 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12866 */
12867 int rc2 = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
12868 AssertRCReturn(rc2, rc2);
12869
12870 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12871 * execution engines about whether hyper BPs and such are pending. */
12872 uint32_t const uDr7 = pMixedCtx->dr[7];
12873 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12874 && X86_DR7_ANY_RW_IO(uDr7)
12875 && (pMixedCtx->cr4 & X86_CR4_DE))
12876 || DBGFBpIsHwIoArmed(pVM)))
12877 {
12878 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12879
12880 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12881 VMMRZCallRing3Disable(pVCpu);
12882 HM_DISABLE_PREEMPT();
12883
12884 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12885
12886 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12887 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12888 {
12889 /* Raise #DB. */
12890 if (fIsGuestDbgActive)
12891 ASMSetDR6(pMixedCtx->dr[6]);
12892 if (pMixedCtx->dr[7] != uDr7)
12893 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
12894
12895 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12896 }
12897 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12898 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12899 else if ( rcStrict2 != VINF_SUCCESS
12900 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12901 rcStrict = rcStrict2;
12902 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12903
12904 HM_RESTORE_PREEMPT();
12905 VMMRZCallRing3Enable(pVCpu);
12906 }
12907 }
12908
12909#ifdef VBOX_STRICT
12910 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12911 Assert(!fIOWrite);
12912 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12913 Assert(fIOWrite);
12914 else
12915 {
12916#if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12917 * statuses, that the VMM device and some others may return. See
12918 * IOM_SUCCESS() for guidance. */
12919 AssertMsg( RT_FAILURE(rcStrict)
12920 || rcStrict == VINF_SUCCESS
12921 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12922 || rcStrict == VINF_EM_DBG_BREAKPOINT
12923 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12924 || rcStrict == VINF_EM_RAW_TO_R3
12925 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12926#endif
12927 }
12928#endif
12929
12930 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12931 return rcStrict;
12932}
12933
12934
12935/**
12936 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12937 * VM-exit.
12938 */
12939HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12940{
12941 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12942
12943 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12944 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12945 AssertRCReturn(rc, rc);
12946 if (VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_TASK_SWITCH_TYPE_IDT)
12947 {
12948 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12949 AssertRCReturn(rc, rc);
12950 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12951 {
12952 uint32_t uErrCode;
12953 RTGCUINTPTR GCPtrFaultAddress;
12954 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12955 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12956 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12957 if (fErrorCodeValid)
12958 {
12959 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12960 AssertRCReturn(rc, rc);
12961 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12962 }
12963 else
12964 uErrCode = 0;
12965
12966 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12967 && uVector == X86_XCPT_PF)
12968 GCPtrFaultAddress = pMixedCtx->cr2;
12969 else
12970 GCPtrFaultAddress = 0;
12971
12972 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12973 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12974
12975 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12976 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12977 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12978 }
12979 }
12980
12981 /* Fall back to the interpreter to emulate the task-switch. */
12982 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12983 return VERR_EM_INTERPRETER;
12984}
12985
12986
12987/**
12988 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12989 */
12990HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12991{
12992 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12993 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12994 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12995 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12996 AssertRCReturn(rc, rc);
12997 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12998 return VINF_EM_DBG_STEPPED;
12999}
13000
13001
13002/**
13003 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
13004 */
13005HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13006{
13007 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13008
13009 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
13010
13011 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13012 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13013 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13014 {
13015 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
13016 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13017 {
13018 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13019 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13020 }
13021 }
13022 else
13023 {
13024 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13025 rcStrict1 = VINF_SUCCESS;
13026 return rcStrict1;
13027 }
13028
13029#if 0
13030 /** @todo Investigate if IOMMMIOPhysHandler() requires a lot of state, for now
13031 * just sync the whole thing. */
13032 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13033#else
13034 /* Aggressive state sync. for now. */
13035 int rc = hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13036 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13037 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13038#endif
13039 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13040 AssertRCReturn(rc, rc);
13041
13042 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
13043 uint32_t uAccessType = VMX_EXIT_QUALIFICATION_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
13044 VBOXSTRICTRC rcStrict2;
13045 switch (uAccessType)
13046 {
13047 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
13048 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
13049 {
13050 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
13051 || VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
13052 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
13053
13054 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
13055 GCPhys &= PAGE_BASE_GC_MASK;
13056 GCPhys += VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
13057 PVM pVM = pVCpu->CTX_SUFF(pVM);
13058 Log4(("ApicAccess uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
13059 VMX_EXIT_QUALIFICATION_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
13060
13061 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
13062 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
13063 CPUMCTX2CORE(pMixedCtx), GCPhys);
13064 Log4(("ApicAccess rcStrict2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
13065 if ( rcStrict2 == VINF_SUCCESS
13066 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13067 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13068 {
13069 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13070 | HM_CHANGED_GUEST_RSP
13071 | HM_CHANGED_GUEST_RFLAGS
13072 | HM_CHANGED_GUEST_APIC_STATE);
13073 rcStrict2 = VINF_SUCCESS;
13074 }
13075 break;
13076 }
13077
13078 default:
13079 Log4(("ApicAccess uAccessType=%#x\n", uAccessType));
13080 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
13081 break;
13082 }
13083
13084 if (rcStrict2 != VINF_SUCCESS)
13085 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
13086 return rcStrict2;
13087}
13088
13089
13090/**
13091 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
13092 * VM-exit.
13093 */
13094HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13095{
13096 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13097
13098 /* We should -not- get this VM-exit if the guest's debug registers were active. */
13099 if (pVmxTransient->fWasGuestDebugStateActive)
13100 {
13101 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
13102 HMVMX_RETURN_UNEXPECTED_EXIT();
13103 }
13104
13105 if ( !pVCpu->hm.s.fSingleInstruction
13106 && !pVmxTransient->fWasHyperDebugStateActive)
13107 {
13108 Assert(!DBGFIsStepping(pVCpu));
13109 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
13110
13111 /* Don't intercept MOV DRx any more. */
13112 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
13113 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
13114 AssertRCReturn(rc, rc);
13115
13116 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
13117 VMMRZCallRing3Disable(pVCpu);
13118 HM_DISABLE_PREEMPT();
13119
13120 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
13121 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
13122 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
13123
13124 HM_RESTORE_PREEMPT();
13125 VMMRZCallRing3Enable(pVCpu);
13126
13127#ifdef VBOX_WITH_STATISTICS
13128 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13129 AssertRCReturn(rc, rc);
13130 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13132 else
13133 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13134#endif
13135 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
13136 return VINF_SUCCESS;
13137 }
13138
13139 /*
13140 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
13141 * Update the segment registers and DR7 from the CPU.
13142 */
13143 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13144 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13145 rc |= hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13146 AssertRCReturn(rc, rc);
13147 Log4(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13148
13149 PVM pVM = pVCpu->CTX_SUFF(pVM);
13150 if (VMX_EXIT_QUALIFICATION_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUALIFICATION_DRX_DIRECTION_WRITE)
13151 {
13152 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13153 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification),
13154 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification));
13155 if (RT_SUCCESS(rc))
13156 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_DEBUG);
13157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
13158 }
13159 else
13160 {
13161 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
13162 VMX_EXIT_QUALIFICATION_DRX_GENREG(pVmxTransient->uExitQualification),
13163 VMX_EXIT_QUALIFICATION_DRX_REGISTER(pVmxTransient->uExitQualification));
13164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
13165 }
13166
13167 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
13168 if (RT_SUCCESS(rc))
13169 {
13170 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13171 AssertRCReturn(rc2, rc2);
13172 return VINF_SUCCESS;
13173 }
13174 return rc;
13175}
13176
13177
13178/**
13179 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
13180 * Conditional VM-exit.
13181 */
13182HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13183{
13184 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13185 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13186
13187 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13188 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13189 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13190 {
13191 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
13192 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
13193 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13194 {
13195 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
13196 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13197 }
13198 }
13199 else
13200 {
13201 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13202 rcStrict1 = VINF_SUCCESS;
13203 return rcStrict1;
13204 }
13205
13206 RTGCPHYS GCPhys = 0;
13207 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13208
13209#if 0
13210 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13211#else
13212 /* Aggressive state sync. for now. */
13213 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13214 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13215 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13216#endif
13217 AssertRCReturn(rc, rc);
13218
13219 /*
13220 * If we succeed, resume guest execution.
13221 * If we fail in interpreting the instruction because we couldn't get the guest physical address
13222 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
13223 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
13224 * weird case. See @bugref{6043}.
13225 */
13226 PVM pVM = pVCpu->CTX_SUFF(pVM);
13227 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
13228 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict2)));
13229 if ( rcStrict2 == VINF_SUCCESS
13230 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13231 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13232 {
13233 /* Successfully handled MMIO operation. */
13234 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13235 | HM_CHANGED_GUEST_RSP
13236 | HM_CHANGED_GUEST_RFLAGS
13237 | HM_CHANGED_GUEST_APIC_STATE);
13238 return VINF_SUCCESS;
13239 }
13240 return rcStrict2;
13241}
13242
13243
13244/**
13245 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
13246 * VM-exit.
13247 */
13248HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13249{
13250 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
13251 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
13252
13253 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
13254 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
13255 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
13256 {
13257 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
13258 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
13259 Log4(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
13260 }
13261 else
13262 {
13263 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
13264 rcStrict1 = VINF_SUCCESS;
13265 return rcStrict1;
13266 }
13267
13268 RTGCPHYS GCPhys = 0;
13269 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
13270 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13271#if 0
13272 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx); /** @todo Can we do better? */
13273#else
13274 /* Aggressive state sync. for now. */
13275 rc |= hmR0VmxSaveGuestRipRspRflags(pVCpu, pMixedCtx);
13276 rc |= hmR0VmxSaveGuestControlRegs(pVCpu, pMixedCtx);
13277 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13278#endif
13279 AssertRCReturn(rc, rc);
13280
13281 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
13282 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
13283
13284 RTGCUINT uErrorCode = 0;
13285 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_INSTR_FETCH)
13286 uErrorCode |= X86_TRAP_PF_ID;
13287 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_DATA_WRITE)
13288 uErrorCode |= X86_TRAP_PF_RW;
13289 if (pVmxTransient->uExitQualification & VMX_EXIT_QUALIFICATION_EPT_ENTRY_PRESENT)
13290 uErrorCode |= X86_TRAP_PF_P;
13291
13292 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
13293
13294 Log4(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
13295 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13296
13297 /* Handle the pagefault trap for the nested shadow table. */
13298 PVM pVM = pVCpu->CTX_SUFF(pVM);
13299 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
13300 TRPMResetTrap(pVCpu);
13301
13302 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
13303 if ( rcStrict2 == VINF_SUCCESS
13304 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
13305 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
13306 {
13307 /* Successfully synced our nested page tables. */
13308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
13309 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13310 | HM_CHANGED_GUEST_RSP
13311 | HM_CHANGED_GUEST_RFLAGS);
13312 return VINF_SUCCESS;
13313 }
13314
13315 Log4(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
13316 return rcStrict2;
13317}
13318
13319/** @} */
13320
13321/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13322/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
13323/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
13324
13325/** @name VM-exit exception handlers.
13326 * @{
13327 */
13328
13329/**
13330 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
13331 */
13332static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13333{
13334 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13335 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
13336
13337 int rc = hmR0VmxSaveGuestCR0(pVCpu, pMixedCtx);
13338 AssertRCReturn(rc, rc);
13339
13340 if (!(pMixedCtx->cr0 & X86_CR0_NE))
13341 {
13342 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
13343 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
13344
13345 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
13346 * provides VM-exit instruction length. If this causes problem later,
13347 * disassemble the instruction like it's done on AMD-V. */
13348 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
13349 AssertRCReturn(rc2, rc2);
13350 return rc;
13351 }
13352
13353 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13354 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13355 return rc;
13356}
13357
13358
13359/**
13360 * VM-exit exception handler for \#BP (Breakpoint exception).
13361 */
13362static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13363{
13364 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
13366
13367 /** @todo Try optimize this by not saving the entire guest state unless
13368 * really needed. */
13369 int rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13370 AssertRCReturn(rc, rc);
13371
13372 PVM pVM = pVCpu->CTX_SUFF(pVM);
13373 rc = DBGFRZTrap03Handler(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
13374 if (rc == VINF_EM_RAW_GUEST_TRAP)
13375 {
13376 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13377 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13378 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13379 AssertRCReturn(rc, rc);
13380
13381 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13382 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13383 }
13384
13385 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
13386 return rc;
13387}
13388
13389
13390/**
13391 * VM-exit exception handler for \#AC (alignment check exception).
13392 */
13393static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13394{
13395 RT_NOREF_PV(pMixedCtx);
13396 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13397
13398 /*
13399 * Re-inject it. We'll detect any nesting before getting here.
13400 */
13401 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13402 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13403 AssertRCReturn(rc, rc);
13404 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13405
13406 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13407 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13408 return VINF_SUCCESS;
13409}
13410
13411
13412/**
13413 * VM-exit exception handler for \#DB (Debug exception).
13414 */
13415static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13416{
13417 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13418 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
13419 Log6(("XcptDB\n"));
13420
13421 /*
13422 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
13423 * for processing.
13424 */
13425 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13426 AssertRCReturn(rc, rc);
13427
13428 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
13429 uint64_t uDR6 = X86_DR6_INIT_VAL;
13430 uDR6 |= ( pVmxTransient->uExitQualification
13431 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
13432
13433 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
13434 if (rc == VINF_EM_RAW_GUEST_TRAP)
13435 {
13436 /*
13437 * The exception was for the guest. Update DR6, DR7.GD and
13438 * IA32_DEBUGCTL.LBR before forwarding it.
13439 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
13440 */
13441 VMMRZCallRing3Disable(pVCpu);
13442 HM_DISABLE_PREEMPT();
13443
13444 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
13445 pMixedCtx->dr[6] |= uDR6;
13446 if (CPUMIsGuestDebugStateActive(pVCpu))
13447 ASMSetDR6(pMixedCtx->dr[6]);
13448
13449 HM_RESTORE_PREEMPT();
13450 VMMRZCallRing3Enable(pVCpu);
13451
13452 rc = hmR0VmxSaveGuestDR7(pVCpu, pMixedCtx);
13453 AssertRCReturn(rc, rc);
13454
13455 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
13456 pMixedCtx->dr[7] &= ~X86_DR7_GD;
13457
13458 /* Paranoia. */
13459 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
13460 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
13461
13462 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
13463 AssertRCReturn(rc, rc);
13464
13465 /*
13466 * Raise #DB in the guest.
13467 *
13468 * It is important to reflect what the VM-exit gave us (preserving the interruption-type) rather than use
13469 * hmR0VmxSetPendingXcptDB() as the #DB could've been raised while executing ICEBP (INT1) and not the
13470 * regular #DB. Thus it -may- trigger different handling in the CPU (like skipped DPL checks), see @bugref{6398}.
13471 *
13472 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of Intel 386,
13473 * see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
13474 */
13475 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13476 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13477 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13478 AssertRCReturn(rc, rc);
13479 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13480 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13481 return VINF_SUCCESS;
13482 }
13483
13484 /*
13485 * Not a guest trap, must be a hypervisor related debug event then.
13486 * Update DR6 in case someone is interested in it.
13487 */
13488 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13489 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13490 CPUMSetHyperDR6(pVCpu, uDR6);
13491
13492 return rc;
13493}
13494
13495/**
13496 * VM-exit exception handler for \#GP (General-protection exception).
13497 *
13498 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13499 */
13500static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13501{
13502 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13503 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13504
13505 int rc;
13506 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13507 { /* likely */ }
13508 else
13509 {
13510#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13511 Assert(pVCpu->hm.s.fUsingDebugLoop);
13512#endif
13513 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13514 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13515 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13516 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13517 rc |= hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13518 AssertRCReturn(rc, rc);
13519 Log4(("#GP Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13520 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13521 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13522 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13523 return rc;
13524 }
13525
13526 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13527 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13528
13529 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13530 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13531 AssertRCReturn(rc, rc);
13532
13533 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13534 uint32_t cbOp = 0;
13535 PVM pVM = pVCpu->CTX_SUFF(pVM);
13536 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13537 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13538 if (RT_SUCCESS(rc))
13539 {
13540 rc = VINF_SUCCESS;
13541 Assert(cbOp == pDis->cbInstr);
13542 Log4(("#GP Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13543 switch (pDis->pCurInstr->uOpcode)
13544 {
13545 case OP_CLI:
13546 {
13547 pMixedCtx->eflags.Bits.u1IF = 0;
13548 pMixedCtx->eflags.Bits.u1RF = 0;
13549 pMixedCtx->rip += pDis->cbInstr;
13550 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13551 if ( !fDbgStepping
13552 && pMixedCtx->eflags.Bits.u1TF)
13553 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13554 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13555 break;
13556 }
13557
13558 case OP_STI:
13559 {
13560 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13561 pMixedCtx->eflags.Bits.u1IF = 1;
13562 pMixedCtx->eflags.Bits.u1RF = 0;
13563 pMixedCtx->rip += pDis->cbInstr;
13564 if (!fOldIF)
13565 {
13566 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13567 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13568 }
13569 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13570 if ( !fDbgStepping
13571 && pMixedCtx->eflags.Bits.u1TF)
13572 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13573 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13574 break;
13575 }
13576
13577 case OP_HLT:
13578 {
13579 rc = VINF_EM_HALT;
13580 pMixedCtx->rip += pDis->cbInstr;
13581 pMixedCtx->eflags.Bits.u1RF = 0;
13582 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13584 break;
13585 }
13586
13587 case OP_POPF:
13588 {
13589 Log4(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13590 uint32_t cbParm;
13591 uint32_t uMask;
13592 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13593 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13594 {
13595 cbParm = 4;
13596 uMask = 0xffffffff;
13597 }
13598 else
13599 {
13600 cbParm = 2;
13601 uMask = 0xffff;
13602 }
13603
13604 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13605 RTGCPTR GCPtrStack = 0;
13606 X86EFLAGS Eflags;
13607 Eflags.u32 = 0;
13608 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13609 &GCPtrStack);
13610 if (RT_SUCCESS(rc))
13611 {
13612 Assert(sizeof(Eflags.u32) >= cbParm);
13613 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13614 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13615 }
13616 if (RT_FAILURE(rc))
13617 {
13618 rc = VERR_EM_INTERPRETER;
13619 break;
13620 }
13621 Log4(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13622 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13623 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13624 pMixedCtx->esp += cbParm;
13625 pMixedCtx->esp &= uMask;
13626 pMixedCtx->rip += pDis->cbInstr;
13627 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13628 | HM_CHANGED_GUEST_RSP
13629 | HM_CHANGED_GUEST_RFLAGS);
13630 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13631 POPF restores EFLAGS.TF. */
13632 if ( !fDbgStepping
13633 && fGstStepping)
13634 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13636 break;
13637 }
13638
13639 case OP_PUSHF:
13640 {
13641 uint32_t cbParm;
13642 uint32_t uMask;
13643 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13644 {
13645 cbParm = 4;
13646 uMask = 0xffffffff;
13647 }
13648 else
13649 {
13650 cbParm = 2;
13651 uMask = 0xffff;
13652 }
13653
13654 /* Get the stack pointer & push the contents of eflags onto the stack. */
13655 RTGCPTR GCPtrStack = 0;
13656 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13657 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13658 if (RT_FAILURE(rc))
13659 {
13660 rc = VERR_EM_INTERPRETER;
13661 break;
13662 }
13663 X86EFLAGS Eflags = pMixedCtx->eflags;
13664 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13665 Eflags.Bits.u1RF = 0;
13666 Eflags.Bits.u1VM = 0;
13667
13668 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13669 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13670 {
13671 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13672 rc = VERR_EM_INTERPRETER;
13673 break;
13674 }
13675 Log4(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13676 pMixedCtx->esp -= cbParm;
13677 pMixedCtx->esp &= uMask;
13678 pMixedCtx->rip += pDis->cbInstr;
13679 pMixedCtx->eflags.Bits.u1RF = 0;
13680 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13681 | HM_CHANGED_GUEST_RSP
13682 | HM_CHANGED_GUEST_RFLAGS);
13683 if ( !fDbgStepping
13684 && pMixedCtx->eflags.Bits.u1TF)
13685 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13686 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13687 break;
13688 }
13689
13690 case OP_IRET:
13691 {
13692 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13693 * instruction reference. */
13694 RTGCPTR GCPtrStack = 0;
13695 uint32_t uMask = 0xffff;
13696 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13697 uint16_t aIretFrame[3];
13698 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13699 {
13700 rc = VERR_EM_INTERPRETER;
13701 break;
13702 }
13703 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13704 &GCPtrStack);
13705 if (RT_SUCCESS(rc))
13706 {
13707 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13708 PGMACCESSORIGIN_HM));
13709 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13710 }
13711 if (RT_FAILURE(rc))
13712 {
13713 rc = VERR_EM_INTERPRETER;
13714 break;
13715 }
13716 pMixedCtx->eip = 0;
13717 pMixedCtx->ip = aIretFrame[0];
13718 pMixedCtx->cs.Sel = aIretFrame[1];
13719 pMixedCtx->cs.ValidSel = aIretFrame[1];
13720 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13721 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13722 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13723 pMixedCtx->sp += sizeof(aIretFrame);
13724 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13725 | HM_CHANGED_GUEST_SEGMENT_REGS
13726 | HM_CHANGED_GUEST_RSP
13727 | HM_CHANGED_GUEST_RFLAGS);
13728 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13729 if ( !fDbgStepping
13730 && fGstStepping)
13731 hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13732 Log4(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13734 break;
13735 }
13736
13737 case OP_INT:
13738 {
13739 uint16_t uVector = pDis->Param1.uValue & 0xff;
13740 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13741 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13742 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13743 break;
13744 }
13745
13746 case OP_INTO:
13747 {
13748 if (pMixedCtx->eflags.Bits.u1OF)
13749 {
13750 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13751 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13753 }
13754 else
13755 {
13756 pMixedCtx->eflags.Bits.u1RF = 0;
13757 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RFLAGS);
13758 }
13759 break;
13760 }
13761
13762 default:
13763 {
13764 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13765 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13766 EMCODETYPE_SUPERVISOR);
13767 rc = VBOXSTRICTRC_VAL(rc2);
13768 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13769 /** @todo We have to set pending-debug exceptions here when the guest is
13770 * single-stepping depending on the instruction that was interpreted. */
13771 Log4(("#GP rc=%Rrc\n", rc));
13772 break;
13773 }
13774 }
13775 }
13776 else
13777 rc = VERR_EM_INTERPRETER;
13778
13779 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13780 ("#GP Unexpected rc=%Rrc\n", rc));
13781 return rc;
13782}
13783
13784
13785/**
13786 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13787 * the exception reported in the VMX transient structure back into the VM.
13788 *
13789 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13790 * up-to-date.
13791 */
13792static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13793{
13794 RT_NOREF_PV(pMixedCtx);
13795 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13796#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13797 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13798 ("uVector=%#04x u32XcptBitmap=%#010RX32\n",
13799 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13800#endif
13801
13802 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13803 hmR0VmxCheckExitDueToEventDelivery(). */
13804 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13805 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13806 AssertRCReturn(rc, rc);
13807 Assert(pVmxTransient->fVmcsFieldsRead & HMVMX_UPDATED_TRANSIENT_EXIT_INTERRUPTION_INFO);
13808
13809#ifdef DEBUG_ramshankar
13810 rc |= hmR0VmxSaveGuestSegmentRegs(pVCpu, pMixedCtx);
13811 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13812 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13813#endif
13814
13815 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13816 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13817 return VINF_SUCCESS;
13818}
13819
13820
13821/**
13822 * VM-exit exception handler for \#PF (Page-fault exception).
13823 */
13824static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13825{
13826 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13827 PVM pVM = pVCpu->CTX_SUFF(pVM);
13828 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13829 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13830 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13831 AssertRCReturn(rc, rc);
13832
13833 if (!pVM->hm.s.fNestedPaging)
13834 { /* likely */ }
13835 else
13836 {
13837#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13838 Assert(pVCpu->hm.s.fUsingDebugLoop);
13839#endif
13840 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13841 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13842 {
13843 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13844 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13845 }
13846 else
13847 {
13848 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13849 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13850 Log4(("Pending #DF due to vectoring #PF. NP\n"));
13851 }
13852 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13853 return rc;
13854 }
13855
13856 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13857 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13858 if (pVmxTransient->fVectoringPF)
13859 {
13860 Assert(pVCpu->hm.s.Event.fPending);
13861 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13862 }
13863
13864 rc = hmR0VmxSaveGuestState(pVCpu, pMixedCtx);
13865 AssertRCReturn(rc, rc);
13866
13867 Log4(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13868 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13869
13870 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13871 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13872 (RTGCPTR)pVmxTransient->uExitQualification);
13873
13874 Log4(("#PF: rc=%Rrc\n", rc));
13875 if (rc == VINF_SUCCESS)
13876 {
13877#if 0
13878 /* Successfully synced shadow pages tables or emulated an MMIO instruction. */
13879 /** @todo this isn't quite right, what if guest does lgdt with some MMIO
13880 * memory? We don't update the whole state here... */
13881 HMCPU_CF_SET(pVCpu, HM_CHANGED_GUEST_RIP
13882 | HM_CHANGED_GUEST_RSP
13883 | HM_CHANGED_GUEST_RFLAGS
13884 | HM_CHANGED_GUEST_APIC_STATE);
13885#else
13886 /*
13887 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13888 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13889 */
13890 /** @todo take advantage of CPUM changed flags instead of brute forcing. */
13891 HMCPU_CF_SET(pVCpu, HM_CHANGED_ALL_GUEST);
13892#endif
13893 TRPMResetTrap(pVCpu);
13894 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13895 return rc;
13896 }
13897
13898 if (rc == VINF_EM_RAW_GUEST_TRAP)
13899 {
13900 if (!pVmxTransient->fVectoringDoublePF)
13901 {
13902 /* It's a guest page fault and needs to be reflected to the guest. */
13903 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13904 TRPMResetTrap(pVCpu);
13905 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13906 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13907 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13908 }
13909 else
13910 {
13911 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13912 TRPMResetTrap(pVCpu);
13913 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13914 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13915 Log4(("#PF: Pending #DF due to vectoring #PF\n"));
13916 }
13917
13918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13919 return VINF_SUCCESS;
13920 }
13921
13922 TRPMResetTrap(pVCpu);
13923 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13924 return rc;
13925}
13926
13927/** @} */
13928
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette