VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72825

Last change on this file since 72825 was 72825, checked in by vboxsync, 7 years ago

VMM/HMVMXR0: Fix TSC offsetting regression from r123352.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 573.2 KB
Line 
1/* $Id: HMVMXR0.cpp 72825 2018-07-03 14:36:06Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
573 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
574 * VERR_VMX_INVALID_VMCS_FIELD.
575 * @param rc The error code.
576 */
577static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
578{
579 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
580 || rc == VERR_VMX_UNABLE_TO_START_VM)
581 {
582 AssertPtrReturnVoid(pVCpu);
583 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
584 }
585 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
586}
587
588
589/**
590 * Reads the VM-entry interruption-information field from the VMCS into the VMX
591 * transient structure.
592 *
593 * @returns VBox status code.
594 * @param pVmxTransient Pointer to the VMX transient structure.
595 *
596 * @remarks No-long-jump zone!!!
597 */
598DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
599{
600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
601 AssertRCReturn(rc, rc);
602 return VINF_SUCCESS;
603}
604
605#ifdef VBOX_STRICT
606/**
607 * Reads the VM-entry exception error code field from the VMCS into
608 * the VMX transient structure.
609 *
610 * @returns VBox status code.
611 * @param pVmxTransient Pointer to the VMX transient structure.
612 *
613 * @remarks No-long-jump zone!!!
614 */
615DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
616{
617 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
618 AssertRCReturn(rc, rc);
619 return VINF_SUCCESS;
620}
621
622
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641/**
642 * Reads the VM-exit interruption-information field from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
653 AssertRCReturn(rc,rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit interruption error code from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction length field from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction-information field from the VMCS into
700 * the VMX transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the exit code qualification from the VMCS into the VMX transient
719 * structure.
720 *
721 * @returns VBox status code.
722 * @param pVCpu The cross context virtual CPU structure of the
723 * calling EMT. (Required for the VMCS cache case.)
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
729 {
730 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Reads the IDT-vectoring information field from the VMCS into the VMX
740 * transient structure.
741 *
742 * @returns VBox status code.
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 *
745 * @remarks No-long-jump zone!!!
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Reads the IDT-vectoring error code from the VMCS into the VMX
761 * transient structure.
762 *
763 * @returns VBox status code.
764 * @param pVmxTransient Pointer to the VMX transient structure.
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Enters VMX root mode operation on the current CPU.
780 *
781 * @returns VBox status code.
782 * @param pVM The cross context VM structure. Can be
783 * NULL, after a resume.
784 * @param HCPhysCpuPage Physical address of the VMXON region.
785 * @param pvCpuPage Pointer to the VMXON region.
786 */
787static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
788{
789 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
790 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
791 Assert(pvCpuPage);
792 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793
794 if (pVM)
795 {
796 /* Write the VMCS revision dword to the VMXON region. */
797 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
798 }
799
800 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
801 RTCCUINTREG fEFlags = ASMIntDisableFlags();
802
803 /* Enable the VMX bit in CR4 if necessary. */
804 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
805
806 /* Enter VMX root mode. */
807 int rc = VMXEnable(HCPhysCpuPage);
808 if (RT_FAILURE(rc))
809 {
810 if (!(uOldCr4 & X86_CR4_VMXE))
811 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
812
813 if (pVM)
814 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
815 }
816
817 /* Restore interrupts. */
818 ASMSetFlags(fEFlags);
819 return rc;
820}
821
822
823/**
824 * Exits VMX root mode operation on the current CPU.
825 *
826 * @returns VBox status code.
827 */
828static int hmR0VmxLeaveRootMode(void)
829{
830 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
831
832 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
833 RTCCUINTREG fEFlags = ASMIntDisableFlags();
834
835 /* If we're for some reason not in VMX root mode, then don't leave it. */
836 RTCCUINTREG uHostCR4 = ASMGetCR4();
837
838 int rc;
839 if (uHostCR4 & X86_CR4_VMXE)
840 {
841 /* Exit VMX root mode and clear the VMX bit in CR4. */
842 VMXDisable();
843 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
844 rc = VINF_SUCCESS;
845 }
846 else
847 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
848
849 /* Restore interrupts. */
850 ASMSetFlags(fEFlags);
851 return rc;
852}
853
854
855/**
856 * Allocates and maps one physically contiguous page. The allocated page is
857 * zero'd out. (Used by various VT-x structures).
858 *
859 * @returns IPRT status code.
860 * @param pMemObj Pointer to the ring-0 memory object.
861 * @param ppVirt Where to store the virtual address of the
862 * allocation.
863 * @param pHCPhys Where to store the physical address of the
864 * allocation.
865 */
866static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
867{
868 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
869 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
871
872 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
873 if (RT_FAILURE(rc))
874 return rc;
875 *ppVirt = RTR0MemObjAddress(*pMemObj);
876 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
877 ASMMemZero32(*ppVirt, PAGE_SIZE);
878 return VINF_SUCCESS;
879}
880
881
882/**
883 * Frees and unmaps an allocated physical page.
884 *
885 * @param pMemObj Pointer to the ring-0 memory object.
886 * @param ppVirt Where to re-initialize the virtual address of
887 * allocation as 0.
888 * @param pHCPhys Where to re-initialize the physical address of the
889 * allocation as 0.
890 */
891static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
892{
893 AssertPtr(pMemObj);
894 AssertPtr(ppVirt);
895 AssertPtr(pHCPhys);
896 if (*pMemObj != NIL_RTR0MEMOBJ)
897 {
898 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
899 AssertRC(rc);
900 *pMemObj = NIL_RTR0MEMOBJ;
901 *ppVirt = 0;
902 *pHCPhys = 0;
903 }
904}
905
906
907/**
908 * Worker function to free VT-x related structures.
909 *
910 * @returns IPRT status code.
911 * @param pVM The cross context VM structure.
912 */
913static void hmR0VmxStructsFree(PVM pVM)
914{
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 AssertPtr(pVCpu);
919
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
922
923 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
925
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
927 }
928
929 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
930#ifdef VBOX_WITH_CRASHDUMP_MAGIC
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
932#endif
933}
934
935
936/**
937 * Worker function to allocate VT-x related VM structures.
938 *
939 * @returns IPRT status code.
940 * @param pVM The cross context VM structure.
941 */
942static int hmR0VmxStructsAlloc(PVM pVM)
943{
944 /*
945 * Initialize members up-front so we can cleanup properly on allocation failure.
946 */
947#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVM->hm.s.vmx.HCPhys##a_Name = 0;
951
952#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
953 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
954 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
955 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
956
957#ifdef VBOX_WITH_CRASHDUMP_MAGIC
958 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
959#endif
960 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
961
962 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
963 for (VMCPUID i = 0; i < pVM->cCpus; i++)
964 {
965 PVMCPU pVCpu = &pVM->aCpus[i];
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
970 }
971#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
972#undef VMXLOCAL_INIT_VM_MEMOBJ
973
974 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
975 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
976 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
977 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
978
979 /*
980 * Allocate all the VT-x structures.
981 */
982 int rc = VINF_SUCCESS;
983#ifdef VBOX_WITH_CRASHDUMP_MAGIC
984 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
988 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
989#endif
990
991 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
992 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
993 {
994 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
995 &pVM->hm.s.vmx.HCPhysApicAccess);
996 if (RT_FAILURE(rc))
997 goto cleanup;
998 }
999
1000 /*
1001 * Initialize per-VCPU VT-x structures.
1002 */
1003 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1004 {
1005 PVMCPU pVCpu = &pVM->aCpus[i];
1006 AssertPtr(pVCpu);
1007
1008 /* Allocate the VM control structure (VMCS). */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012
1013 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1014 if ( PDMHasApic(pVM)
1015 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1016 {
1017 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1018 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1019 if (RT_FAILURE(rc))
1020 goto cleanup;
1021 }
1022
1023 /*
1024 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1025 * transparent accesses of specific MSRs.
1026 *
1027 * If the condition for enabling MSR bitmaps changes here, don't forget to
1028 * update HMAreMsrBitmapsAvailable().
1029 */
1030 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1031 {
1032 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1033 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1034 if (RT_FAILURE(rc))
1035 goto cleanup;
1036 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1037 }
1038
1039 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043
1044 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1045 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1046 if (RT_FAILURE(rc))
1047 goto cleanup;
1048 }
1049
1050 return VINF_SUCCESS;
1051
1052cleanup:
1053 hmR0VmxStructsFree(pVM);
1054 return rc;
1055}
1056
1057
1058/**
1059 * Does global VT-x initialization (called during module initialization).
1060 *
1061 * @returns VBox status code.
1062 */
1063VMMR0DECL(int) VMXR0GlobalInit(void)
1064{
1065#ifdef HMVMX_USE_FUNCTION_TABLE
1066 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1067# ifdef VBOX_STRICT
1068 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1069 Assert(g_apfnVMExitHandlers[i]);
1070# endif
1071#endif
1072 return VINF_SUCCESS;
1073}
1074
1075
1076/**
1077 * Does global VT-x termination (called during module termination).
1078 */
1079VMMR0DECL(void) VMXR0GlobalTerm()
1080{
1081 /* Nothing to do currently. */
1082}
1083
1084
1085/**
1086 * Sets up and activates VT-x on the current CPU.
1087 *
1088 * @returns VBox status code.
1089 * @param pHostCpu Pointer to the global CPU info struct.
1090 * @param pVM The cross context VM structure. Can be
1091 * NULL after a host resume operation.
1092 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1093 * fEnabledByHost is @c true).
1094 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1095 * @a fEnabledByHost is @c true).
1096 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1097 * enable VT-x on the host.
1098 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1099 */
1100VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1101 void *pvMsrs)
1102{
1103 Assert(pHostCpu);
1104 Assert(pvMsrs);
1105 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1106
1107 /* Enable VT-x if it's not already enabled by the host. */
1108 if (!fEnabledByHost)
1109 {
1110 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1111 if (RT_FAILURE(rc))
1112 return rc;
1113 }
1114
1115 /*
1116 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1117 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1118 * invalidated when flushing by VPID.
1119 */
1120 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1121 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1122 {
1123 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1124 pHostCpu->fFlushAsidBeforeUse = false;
1125 }
1126 else
1127 pHostCpu->fFlushAsidBeforeUse = true;
1128
1129 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1130 ++pHostCpu->cTlbFlushes;
1131
1132 return VINF_SUCCESS;
1133}
1134
1135
1136/**
1137 * Deactivates VT-x on the current CPU.
1138 *
1139 * @returns VBox status code.
1140 * @param pHostCpu Pointer to the global CPU info struct.
1141 * @param pvCpuPage Pointer to the VMXON region.
1142 * @param HCPhysCpuPage Physical address of the VMXON region.
1143 *
1144 * @remarks This function should never be called when SUPR0EnableVTx() or
1145 * similar was used to enable VT-x on the host.
1146 */
1147VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1148{
1149 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1150
1151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1152 return hmR0VmxLeaveRootMode();
1153}
1154
1155
1156/**
1157 * Sets the permission bits for the specified MSR in the MSR bitmap.
1158 *
1159 * @param pVCpu The cross context virtual CPU structure.
1160 * @param uMsr The MSR value.
1161 * @param enmRead Whether reading this MSR causes a VM-exit.
1162 * @param enmWrite Whether writing this MSR causes a VM-exit.
1163 */
1164static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1165{
1166 int32_t iBit;
1167 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1168
1169 /*
1170 * Layout:
1171 * 0x000 - 0x3ff - Low MSR read bits
1172 * 0x400 - 0x7ff - High MSR read bits
1173 * 0x800 - 0xbff - Low MSR write bits
1174 * 0xc00 - 0xfff - High MSR write bits
1175 */
1176 if (uMsr <= 0x00001FFF)
1177 iBit = uMsr;
1178 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1179 {
1180 iBit = uMsr - UINT32_C(0xC0000000);
1181 pbMsrBitmap += 0x400;
1182 }
1183 else
1184 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1185
1186 Assert(iBit <= 0x1fff);
1187 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1188 ASMBitSet(pbMsrBitmap, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap, iBit);
1191
1192 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1193 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1194 else
1195 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1196}
1197
1198
1199#ifdef VBOX_STRICT
1200/**
1201 * Gets the permission bits for the specified MSR in the MSR bitmap.
1202 *
1203 * @returns VBox status code.
1204 * @retval VINF_SUCCESS if the specified MSR is found.
1205 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1206 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1207 *
1208 * @param pVCpu The cross context virtual CPU structure.
1209 * @param uMsr The MSR.
1210 * @param penmRead Where to store the read permissions.
1211 * @param penmWrite Where to store the write permissions.
1212 */
1213static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1214{
1215 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1216 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1217 int32_t iBit;
1218 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1219
1220 /* See hmR0VmxSetMsrPermission() for the layout. */
1221 if (uMsr <= 0x00001FFF)
1222 iBit = uMsr;
1223 else if ( uMsr >= 0xC0000000
1224 && uMsr <= 0xC0001FFF)
1225 {
1226 iBit = (uMsr - 0xC0000000);
1227 pbMsrBitmap += 0x400;
1228 }
1229 else
1230 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1231
1232 Assert(iBit <= 0x1fff);
1233 if (ASMBitTest(pbMsrBitmap, iBit))
1234 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1235 else
1236 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1237
1238 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1239 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1240 else
1241 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1242 return VINF_SUCCESS;
1243}
1244#endif /* VBOX_STRICT */
1245
1246
1247/**
1248 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1249 * area.
1250 *
1251 * @returns VBox status code.
1252 * @param pVCpu The cross context virtual CPU structure.
1253 * @param cMsrs The number of MSRs.
1254 */
1255static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1256{
1257 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1258 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1259 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1260 {
1261 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1262 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1264 }
1265
1266 /* Update number of guest MSRs to load/store across the world-switch. */
1267 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1269
1270 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1271 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1272 AssertRCReturn(rc, rc);
1273
1274 /* Update the VCPU's copy of the MSR count. */
1275 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1276
1277 return VINF_SUCCESS;
1278}
1279
1280
1281/**
1282 * Adds a new (or updates the value of an existing) guest/host MSR
1283 * pair to be swapped during the world-switch as part of the
1284 * auto-load/store MSR area in the VMCS.
1285 *
1286 * @returns VBox status code.
1287 * @param pVCpu The cross context virtual CPU structure.
1288 * @param uMsr The MSR.
1289 * @param uGuestMsrValue Value of the guest MSR.
1290 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1291 * necessary.
1292 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1293 * its value was updated. Optional, can be NULL.
1294 */
1295static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1296 bool *pfAddedAndUpdated)
1297{
1298 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1299 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1300 uint32_t i;
1301 for (i = 0; i < cMsrs; i++)
1302 {
1303 if (pGuestMsr->u32Msr == uMsr)
1304 break;
1305 pGuestMsr++;
1306 }
1307
1308 bool fAdded = false;
1309 if (i == cMsrs)
1310 {
1311 ++cMsrs;
1312 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1313 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1314
1315 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1316 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1317 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1318
1319 fAdded = true;
1320 }
1321
1322 /* Update the MSR values in the auto-load/store MSR area. */
1323 pGuestMsr->u32Msr = uMsr;
1324 pGuestMsr->u64Value = uGuestMsrValue;
1325
1326 /* Create/update the MSR slot in the host MSR area. */
1327 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1328 pHostMsr += i;
1329 pHostMsr->u32Msr = uMsr;
1330
1331 /*
1332 * Update the host MSR only when requested by the caller AND when we're
1333 * adding it to the auto-load/store area. Otherwise, it would have been
1334 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1335 */
1336 bool fUpdatedMsrValue = false;
1337 if ( fAdded
1338 && fUpdateHostMsr)
1339 {
1340 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1341 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1342 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1343 fUpdatedMsrValue = true;
1344 }
1345
1346 if (pfAddedAndUpdated)
1347 *pfAddedAndUpdated = fUpdatedMsrValue;
1348 return VINF_SUCCESS;
1349}
1350
1351
1352/**
1353 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1354 * auto-load/store MSR area in the VMCS.
1355 *
1356 * @returns VBox status code.
1357 * @param pVCpu The cross context virtual CPU structure.
1358 * @param uMsr The MSR.
1359 */
1360static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1361{
1362 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1363 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1364 for (uint32_t i = 0; i < cMsrs; i++)
1365 {
1366 /* Find the MSR. */
1367 if (pGuestMsr->u32Msr == uMsr)
1368 {
1369 /* If it's the last MSR, simply reduce the count. */
1370 if (i == cMsrs - 1)
1371 {
1372 --cMsrs;
1373 break;
1374 }
1375
1376 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1377 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 pLastGuestMsr += cMsrs - 1;
1379 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1380 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1381
1382 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1383 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 pLastHostMsr += cMsrs - 1;
1385 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1386 pHostMsr->u64Value = pLastHostMsr->u64Value;
1387 --cMsrs;
1388 break;
1389 }
1390 pGuestMsr++;
1391 }
1392
1393 /* Update the VMCS if the count changed (meaning the MSR was found). */
1394 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1395 {
1396 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1397 AssertRCReturn(rc, rc);
1398
1399 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1400 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1401 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1402
1403 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1404 return VINF_SUCCESS;
1405 }
1406
1407 return VERR_NOT_FOUND;
1408}
1409
1410
1411/**
1412 * Checks if the specified guest MSR is part of the auto-load/store area in
1413 * the VMCS.
1414 *
1415 * @returns true if found, false otherwise.
1416 * @param pVCpu The cross context virtual CPU structure.
1417 * @param uMsr The MSR to find.
1418 */
1419static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1420{
1421 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1422 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1423
1424 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1425 {
1426 if (pGuestMsr->u32Msr == uMsr)
1427 return true;
1428 }
1429 return false;
1430}
1431
1432
1433/**
1434 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1435 *
1436 * @param pVCpu The cross context virtual CPU structure.
1437 *
1438 * @remarks No-long-jump zone!!!
1439 */
1440static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1441{
1442 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1443 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1444 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1445 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1446
1447 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1448 {
1449 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1450
1451 /*
1452 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1453 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1454 */
1455 if (pHostMsr->u32Msr == MSR_K6_EFER)
1456 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1457 else
1458 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1459 }
1460
1461 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1462}
1463
1464
1465/**
1466 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1467 * perform lazy restoration of the host MSRs while leaving VT-x.
1468 *
1469 * @param pVCpu The cross context virtual CPU structure.
1470 *
1471 * @remarks No-long-jump zone!!!
1472 */
1473static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1474{
1475 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1476
1477 /*
1478 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1479 */
1480 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1481 {
1482 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1483#if HC_ARCH_BITS == 64
1484 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1485 {
1486 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1487 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1488 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1489 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1490 }
1491#endif
1492 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1493 }
1494}
1495
1496
1497/**
1498 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1499 * lazily while leaving VT-x.
1500 *
1501 * @returns true if it does, false otherwise.
1502 * @param pVCpu The cross context virtual CPU structure.
1503 * @param uMsr The MSR to check.
1504 */
1505static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1506{
1507 NOREF(pVCpu);
1508#if HC_ARCH_BITS == 64
1509 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1510 {
1511 switch (uMsr)
1512 {
1513 case MSR_K8_LSTAR:
1514 case MSR_K6_STAR:
1515 case MSR_K8_SF_MASK:
1516 case MSR_K8_KERNEL_GS_BASE:
1517 return true;
1518 }
1519 }
1520#else
1521 RT_NOREF(pVCpu, uMsr);
1522#endif
1523 return false;
1524}
1525
1526
1527/**
1528 * Loads a set of guests MSRs to allow read/passthru to the guest.
1529 *
1530 * The name of this function is slightly confusing. This function does NOT
1531 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1532 * common prefix for functions dealing with "lazy restoration" of the shared
1533 * MSRs.
1534 *
1535 * @param pVCpu The cross context virtual CPU structure.
1536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1537 * out-of-sync. Make sure to update the required fields
1538 * before using them.
1539 *
1540 * @remarks No-long-jump zone!!!
1541 */
1542static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1543{
1544 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1545 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1546
1547 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1548#if HC_ARCH_BITS == 64
1549 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1550 {
1551 /*
1552 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1553 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1554 * we can skip a few MSR writes.
1555 *
1556 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1557 * guest MSR values in the guest-CPU context might be different to what's currently
1558 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1559 * CPU, see @bugref{8728}.
1560 */
1561 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1562 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1563 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1564 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1565 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1566 {
1567#ifdef VBOX_STRICT
1568 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1569 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1570 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1571 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1572#endif
1573 }
1574 else
1575 {
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1577 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1578 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1579 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1580 }
1581 }
1582#else
1583 RT_NOREF(pMixedCtx);
1584#endif
1585 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1586}
1587
1588
1589/**
1590 * Performs lazy restoration of the set of host MSRs if they were previously
1591 * loaded with guest MSR values.
1592 *
1593 * @param pVCpu The cross context virtual CPU structure.
1594 *
1595 * @remarks No-long-jump zone!!!
1596 * @remarks The guest MSRs should have been saved back into the guest-CPU
1597 * context by hmR0VmxImportGuestState()!!!
1598 */
1599static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1600{
1601 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1603
1604 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1605 {
1606 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1607#if HC_ARCH_BITS == 64
1608 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1609 {
1610 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1611 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1612 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1613 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1614 }
1615#endif
1616 }
1617 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1618}
1619
1620
1621/**
1622 * Verifies that our cached values of the VMCS controls are all
1623 * consistent with what's actually present in the VMCS.
1624 *
1625 * @returns VBox status code.
1626 * @param pVCpu The cross context virtual CPU structure.
1627 */
1628static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1629{
1630 uint32_t u32Val;
1631 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1632 AssertRCReturn(rc, rc);
1633 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1634 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1639 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1649 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1650
1651 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1652 {
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1656 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1657 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1658 }
1659
1660 return VINF_SUCCESS;
1661}
1662
1663
1664#ifdef VBOX_STRICT
1665/**
1666 * Verifies that our cached host EFER value has not changed
1667 * since we cached it.
1668 *
1669 * @param pVCpu The cross context virtual CPU structure.
1670 */
1671static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1672{
1673 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1674
1675 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1676 {
1677 uint64_t u64Val;
1678 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1679 AssertRC(rc);
1680
1681 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1682 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1683 }
1684}
1685
1686
1687/**
1688 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1689 * VMCS are correct.
1690 *
1691 * @param pVCpu The cross context virtual CPU structure.
1692 */
1693static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1694{
1695 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1696
1697 /* Verify MSR counts in the VMCS are what we think it should be. */
1698 uint32_t cMsrs;
1699 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1700 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1701
1702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1703 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1704
1705 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1709 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1710 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1711 {
1712 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1713 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1714 pGuestMsr->u32Msr, cMsrs));
1715
1716 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1717 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1718 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1719
1720 /* Verify that the permissions are as expected in the MSR bitmap. */
1721 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1722 {
1723 VMXMSREXITREAD enmRead;
1724 VMXMSREXITWRITE enmWrite;
1725 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1726 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1727 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1728 {
1729 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1730 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1731 }
1732 else
1733 {
1734 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1735 pGuestMsr->u32Msr, cMsrs));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1737 pGuestMsr->u32Msr, cMsrs));
1738 }
1739 }
1740 }
1741}
1742#endif /* VBOX_STRICT */
1743
1744
1745/**
1746 * Flushes the TLB using EPT.
1747 *
1748 * @returns VBox status code.
1749 * @param pVCpu The cross context virtual CPU structure of the calling
1750 * EMT. Can be NULL depending on @a enmFlush.
1751 * @param enmFlush Type of flush.
1752 *
1753 * @remarks Caller is responsible for making sure this function is called only
1754 * when NestedPaging is supported and providing @a enmFlush that is
1755 * supported by the CPU.
1756 * @remarks Can be called with interrupts disabled.
1757 */
1758static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1759{
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1762 au64Descriptor[0] = 0;
1763 else
1764 {
1765 Assert(pVCpu);
1766 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1767 }
1768 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1769
1770 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1771 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1772 rc));
1773 if ( RT_SUCCESS(rc)
1774 && pVCpu)
1775 {
1776 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1777 }
1778}
1779
1780
1781/**
1782 * Flushes the TLB using VPID.
1783 *
1784 * @returns VBox status code.
1785 * @param pVCpu The cross context virtual CPU structure of the calling
1786 * EMT. Can be NULL depending on @a enmFlush.
1787 * @param enmFlush Type of flush.
1788 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1789 * on @a enmFlush).
1790 *
1791 * @remarks Can be called with interrupts disabled.
1792 */
1793static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1794{
1795 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1796
1797 uint64_t au64Descriptor[2];
1798 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1799 {
1800 au64Descriptor[0] = 0;
1801 au64Descriptor[1] = 0;
1802 }
1803 else
1804 {
1805 AssertPtr(pVCpu);
1806 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1807 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1808 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1809 au64Descriptor[1] = GCPtr;
1810 }
1811
1812 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1813 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1814 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1815 if ( RT_SUCCESS(rc)
1816 && pVCpu)
1817 {
1818 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1819 }
1820 NOREF(rc);
1821}
1822
1823
1824/**
1825 * Invalidates a guest page by guest virtual address. Only relevant for
1826 * EPT/VPID, otherwise there is nothing really to invalidate.
1827 *
1828 * @returns VBox status code.
1829 * @param pVCpu The cross context virtual CPU structure.
1830 * @param GCVirt Guest virtual address of the page to invalidate.
1831 */
1832VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1833{
1834 AssertPtr(pVCpu);
1835 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1836
1837 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1838 if (!fFlushPending)
1839 {
1840 /*
1841 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1842 * the EPT case. See @bugref{6043} and @bugref{6177}.
1843 *
1844 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1845 * as this function maybe called in a loop with individual addresses.
1846 */
1847 PVM pVM = pVCpu->CTX_SUFF(pVM);
1848 if (pVM->hm.s.vmx.fVpid)
1849 {
1850 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1851
1852#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1853 /*
1854 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1855 * where executing INVVPID outside 64-bit mode does not flush translations of
1856 * 64-bit linear addresses, see @bugref{6208#c72}.
1857 */
1858 if (RT_HI_U32(GCVirt))
1859 fVpidFlush = false;
1860#endif
1861
1862 if (fVpidFlush)
1863 {
1864 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1880 * case where neither EPT nor VPID is supported by the CPU.
1881 *
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param pCpu Pointer to the global HM struct.
1884 *
1885 * @remarks Called with interrupts disabled.
1886 */
1887static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1888{
1889 AssertPtr(pVCpu);
1890 AssertPtr(pCpu);
1891
1892 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1893
1894 Assert(pCpu->idCpu != NIL_RTCPUID);
1895 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1896 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1897 pVCpu->hm.s.fForceTLBFlush = false;
1898 return;
1899}
1900
1901
1902/**
1903 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1904 *
1905 * @param pVCpu The cross context virtual CPU structure.
1906 * @param pCpu Pointer to the global HM CPU struct.
1907 *
1908 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1909 * nomenclature. The reason is, to avoid confusion in compare statements
1910 * since the host-CPU copies are named "ASID".
1911 *
1912 * @remarks Called with interrupts disabled.
1913 */
1914static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915{
1916#ifdef VBOX_WITH_STATISTICS
1917 bool fTlbFlushed = false;
1918# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1919# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1920 if (!fTlbFlushed) \
1921 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1922 } while (0)
1923#else
1924# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1925# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1926#endif
1927
1928 AssertPtr(pCpu);
1929 AssertPtr(pVCpu);
1930 Assert(pCpu->idCpu != NIL_RTCPUID);
1931
1932 PVM pVM = pVCpu->CTX_SUFF(pVM);
1933 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1934 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1935 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1939 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1940 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1941 * cannot reuse the current ASID anymore.
1942 */
1943 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1944 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1945 {
1946 ++pCpu->uCurrentAsid;
1947 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1948 {
1949 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1950 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1951 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1952 }
1953
1954 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 /*
1959 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1960 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1966 }
1967
1968 /* Check for explicit TLB flushes. */
1969 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1970 {
1971 /*
1972 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1973 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1974 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1975 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1976 * mappings, see @bugref{6568}.
1977 *
1978 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 }
1984
1985 pVCpu->hm.s.fForceTLBFlush = false;
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVCpu The cross context virtual CPU structure.
2011 * @param pCpu Pointer to the global HM CPU struct.
2012 *
2013 * @remarks Called with interrupts disabled.
2014 */
2015static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2016{
2017 AssertPtr(pVCpu);
2018 AssertPtr(pCpu);
2019 Assert(pCpu->idCpu != NIL_RTCPUID);
2020 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2021 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2022
2023 /*
2024 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2025 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2026 */
2027 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2028 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2029 {
2030 pVCpu->hm.s.fForceTLBFlush = true;
2031 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2032 }
2033
2034 /* Check for explicit TLB flushes. */
2035 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2039 }
2040
2041 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2042 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2043
2044 if (pVCpu->hm.s.fForceTLBFlush)
2045 {
2046 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2047 pVCpu->hm.s.fForceTLBFlush = false;
2048 }
2049}
2050
2051
2052/**
2053 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2054 *
2055 * @returns VBox status code.
2056 * @param pVCpu The cross context virtual CPU structure.
2057 * @param pCpu Pointer to the global HM CPU struct.
2058 *
2059 * @remarks Called with interrupts disabled.
2060 */
2061static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062{
2063 AssertPtr(pVCpu);
2064 AssertPtr(pCpu);
2065 Assert(pCpu->idCpu != NIL_RTCPUID);
2066 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2067 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2068
2069 /*
2070 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2071 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2072 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2073 * cannot reuse the current ASID anymore.
2074 */
2075 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2076 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2077 {
2078 pVCpu->hm.s.fForceTLBFlush = true;
2079 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2080 }
2081
2082 /* Check for explicit TLB flushes. */
2083 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2084 {
2085 /*
2086 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2087 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2088 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2089 * include fExplicitFlush's too) - an obscure corner case.
2090 */
2091 pVCpu->hm.s.fForceTLBFlush = true;
2092 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2093 }
2094
2095 PVM pVM = pVCpu->CTX_SUFF(pVM);
2096 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2097 if (pVCpu->hm.s.fForceTLBFlush)
2098 {
2099 ++pCpu->uCurrentAsid;
2100 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2101 {
2102 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2103 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2104 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2105 }
2106
2107 pVCpu->hm.s.fForceTLBFlush = false;
2108 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2109 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2110 if (pCpu->fFlushAsidBeforeUse)
2111 {
2112 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2113 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2114 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2115 {
2116 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2117 pCpu->fFlushAsidBeforeUse = false;
2118 }
2119 else
2120 {
2121 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2122 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2123 }
2124 }
2125 }
2126
2127 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2128 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2129 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2130 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2131 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2132 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2133 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2134
2135 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2136 AssertRC(rc);
2137}
2138
2139
2140/**
2141 * Flushes the guest TLB entry based on CPU capabilities.
2142 *
2143 * @param pVCpu The cross context virtual CPU structure.
2144 * @param pCpu Pointer to the global HM CPU struct.
2145 */
2146DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2147{
2148#ifdef HMVMX_ALWAYS_FLUSH_TLB
2149 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2150#endif
2151 PVM pVM = pVCpu->CTX_SUFF(pVM);
2152 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2153 {
2154 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2155 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2156 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2158 default:
2159 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2160 break;
2161 }
2162 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2163}
2164
2165
2166/**
2167 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2168 * TLB entries from the host TLB before VM-entry.
2169 *
2170 * @returns VBox status code.
2171 * @param pVM The cross context VM structure.
2172 */
2173static int hmR0VmxSetupTaggedTlb(PVM pVM)
2174{
2175 /*
2176 * Determine optimal flush type for Nested Paging.
2177 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2178 * guest execution (see hmR3InitFinalizeR0()).
2179 */
2180 if (pVM->hm.s.fNestedPaging)
2181 {
2182 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2185 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2186 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2187 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2188 else
2189 {
2190 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2192 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2193 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2194 }
2195
2196 /* Make sure the write-back cacheable memory type for EPT is supported. */
2197 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2198 {
2199 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2200 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2201 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2202 }
2203
2204 /* EPT requires a page-walk length of 4. */
2205 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2206 {
2207 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211 }
2212 else
2213 {
2214 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2215 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219 }
2220
2221 /*
2222 * Determine optimal flush type for VPID.
2223 */
2224 if (pVM->hm.s.vmx.fVpid)
2225 {
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2229 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2230 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2231 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2232 else
2233 {
2234 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2236 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2238 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2240 pVM->hm.s.vmx.fVpid = false;
2241 }
2242 }
2243 else
2244 {
2245 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2246 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2247 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2248 pVM->hm.s.vmx.fVpid = false;
2249 }
2250 }
2251
2252 /*
2253 * Setup the handler for flushing tagged-TLBs.
2254 */
2255 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2256 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2257 else if (pVM->hm.s.fNestedPaging)
2258 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2259 else if (pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2261 else
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Sets up pin-based VM-execution controls in the VMCS.
2269 *
2270 * @returns VBox status code.
2271 * @param pVCpu The cross context virtual CPU structure.
2272 *
2273 * @remarks We don't really care about optimizing vmwrites here as it's done only
2274 * once per VM and hence we don't care about VMCS-field cache comparisons.
2275 */
2276static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2277{
2278 PVM pVM = pVCpu->CTX_SUFF(pVM);
2279 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2280 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2281
2282 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2283 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2284
2285 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2286 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2287
2288 /* Enable the VMX preemption timer. */
2289 if (pVM->hm.s.vmx.fUsePreemptTimer)
2290 {
2291 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2293 }
2294
2295#if 0
2296 /* Enable posted-interrupt processing. */
2297 if (pVM->hm.s.fPostedIntrs)
2298 {
2299 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2300 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2301 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2302 }
2303#endif
2304
2305 if ((fVal & fZap) != fVal)
2306 {
2307 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2308 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2309 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2310 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2311 }
2312
2313 /* Commit it to the VMCS and update our cache. */
2314 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2315 AssertRCReturn(rc, rc);
2316 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2317
2318 return VINF_SUCCESS;
2319}
2320
2321
2322/**
2323 * Sets up secondary processor-based VM-execution controls in the VMCS.
2324 *
2325 * @returns VBox status code.
2326 * @param pVCpu The cross context virtual CPU structure.
2327 *
2328 * @remarks We don't really care about optimizing vmwrites here as it's done only
2329 * once per VM and hence we don't care about VMCS-field cache comparisons.
2330 */
2331static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2332{
2333 PVM pVM = pVCpu->CTX_SUFF(pVM);
2334 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2335 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2336
2337 /* WBINVD causes a VM-exit. */
2338 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2339 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2340
2341 /* Enable EPT (aka nested-paging). */
2342 if (pVM->hm.s.fNestedPaging)
2343 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2344
2345 /*
2346 * Enable the INVPCID instruction if supported by the hardware and we expose
2347 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2348 */
2349 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2350 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2351 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2352
2353 /* Enable VPID. */
2354 if (pVM->hm.s.vmx.fVpid)
2355 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2356
2357 /* Enable Unrestricted guest execution. */
2358 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2359 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2360
2361#if 0
2362 if (pVM->hm.s.fVirtApicRegs)
2363 {
2364 /* Enable APIC-register virtualization. */
2365 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2366 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2367
2368 /* Enable virtual-interrupt delivery. */
2369 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2371 }
2372#endif
2373
2374 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2375 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2376 * done dynamically. */
2377 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2378 {
2379 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2380 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2381 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2382 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2383 AssertRCReturn(rc, rc);
2384 }
2385
2386 /* Enable RDTSCP. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2388 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2389
2390 /* Enable Pause-Loop exiting. */
2391 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2392 && pVM->hm.s.vmx.cPleGapTicks
2393 && pVM->hm.s.vmx.cPleWindowTicks)
2394 {
2395 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2396
2397 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2398 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2399 AssertRCReturn(rc, rc);
2400 }
2401
2402 if ((fVal & fZap) != fVal)
2403 {
2404 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2405 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2406 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2407 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2408 }
2409
2410 /* Commit it to the VMCS and update our cache. */
2411 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2412 AssertRCReturn(rc, rc);
2413 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2414
2415 return VINF_SUCCESS;
2416}
2417
2418
2419/**
2420 * Sets up processor-based VM-execution controls in the VMCS.
2421 *
2422 * @returns VBox status code.
2423 * @param pVCpu The cross context virtual CPU structure.
2424 *
2425 * @remarks We don't really care about optimizing vmwrites here as it's done only
2426 * once per VM and hence we don't care about VMCS-field cache comparisons.
2427 */
2428static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2429{
2430 PVM pVM = pVCpu->CTX_SUFF(pVM);
2431 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2432 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2433
2434 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2435 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2436 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2437 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2438 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2441
2442 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2443 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2444 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2445 {
2446 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2447 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2448 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2449 }
2450
2451 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2452 if (!pVM->hm.s.fNestedPaging)
2453 {
2454 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2455 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2456 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2458 }
2459
2460 /* Use TPR shadowing if supported by the CPU. */
2461 if ( PDMHasApic(pVM)
2462 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2463 {
2464 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2465 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2466 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2467 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2468 AssertRCReturn(rc, rc);
2469
2470 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2471 /* CR8 writes cause a VM-exit based on TPR threshold. */
2472 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2473 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2474 }
2475 else
2476 {
2477 /*
2478 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2479 * Set this control only for 64-bit guests.
2480 */
2481 if (pVM->hm.s.fAllow64BitGuests)
2482 {
2483 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2484 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2485 }
2486 }
2487
2488 /* Use MSR-bitmaps if supported by the CPU. */
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2490 {
2491 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2492
2493 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2494 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2495 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 AssertRCReturn(rc, rc);
2497
2498 /*
2499 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2500 * automatically using dedicated fields in the VMCS.
2501 */
2502 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2503 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507#if HC_ARCH_BITS == 64
2508 /*
2509 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2510 */
2511 if (pVM->hm.s.fAllow64BitGuests)
2512 {
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 }
2518#endif
2519 /*
2520 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2521 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2522 */
2523 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525
2526 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2527 }
2528
2529 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2530 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2531 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2532
2533 if ((fVal & fZap) != fVal)
2534 {
2535 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2536 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2537 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2538 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2539 }
2540
2541 /* Commit it to the VMCS and update our cache. */
2542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2543 AssertRCReturn(rc, rc);
2544 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2545
2546 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2547 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 return hmR0VmxSetupProcCtls2(pVCpu);
2549
2550 /* Sanity check, should not really happen. */
2551 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2552 {
2553 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2559 return VINF_SUCCESS;
2560}
2561
2562
2563/**
2564 * Sets up miscellaneous (everything other than Pin & Processor-based
2565 * VM-execution) control fields in the VMCS.
2566 *
2567 * @returns VBox status code.
2568 * @param pVCpu The cross context virtual CPU structure.
2569 */
2570static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2571{
2572 AssertPtr(pVCpu);
2573
2574 int rc = VERR_GENERAL_FAILURE;
2575
2576 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2577#if 0
2578 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2580 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2581
2582 /*
2583 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2584 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2585 * We thus use the exception bitmap to control it rather than use both.
2586 */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2589
2590 /* All IO & IOIO instructions cause VM-exits. */
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2592 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2593
2594 /* Initialize the MSR-bitmap area. */
2595 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2598 AssertRCReturn(rc, rc);
2599#endif
2600
2601 /* Setup MSR auto-load/store area. */
2602 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2606 AssertRCReturn(rc, rc);
2607
2608 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2609 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2610 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2615 AssertRCReturn(rc, rc);
2616
2617 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2618#if 0
2619 /* Setup debug controls */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2621 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2622 AssertRCReturn(rc, rc);
2623#endif
2624
2625 return rc;
2626}
2627
2628
2629/**
2630 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2631 *
2632 * We shall setup those exception intercepts that don't change during the
2633 * lifetime of the VM here. The rest are done dynamically while loading the
2634 * guest state.
2635 *
2636 * @returns VBox status code.
2637 * @param pVCpu The cross context virtual CPU structure.
2638 */
2639static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2640{
2641 AssertPtr(pVCpu);
2642
2643 uint32_t u32XcptBitmap;
2644
2645 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2646 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2647
2648 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2649 and writes, and because recursive #DBs can cause the CPU hang, we must always
2650 intercept #DB. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2652
2653 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2654 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2655 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2656
2657 /* Commit it to the VMCS. */
2658 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2659 AssertRCReturn(rc, rc);
2660
2661 /* Update our cache of the exception bitmap. */
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 return VINF_SUCCESS;
2664}
2665
2666
2667/**
2668 * Does per-VM VT-x initialization.
2669 *
2670 * @returns VBox status code.
2671 * @param pVM The cross context VM structure.
2672 */
2673VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2674{
2675 LogFlowFunc(("pVM=%p\n", pVM));
2676
2677 int rc = hmR0VmxStructsAlloc(pVM);
2678 if (RT_FAILURE(rc))
2679 {
2680 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2681 return rc;
2682 }
2683
2684 return VINF_SUCCESS;
2685}
2686
2687
2688/**
2689 * Does per-VM VT-x termination.
2690 *
2691 * @returns VBox status code.
2692 * @param pVM The cross context VM structure.
2693 */
2694VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2695{
2696 LogFlowFunc(("pVM=%p\n", pVM));
2697
2698#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2700 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2701#endif
2702 hmR0VmxStructsFree(pVM);
2703 return VINF_SUCCESS;
2704}
2705
2706
2707/**
2708 * Sets up the VM for execution under VT-x.
2709 * This function is only called once per-VM during initialization.
2710 *
2711 * @returns VBox status code.
2712 * @param pVM The cross context VM structure.
2713 */
2714VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2715{
2716 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2717 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2718
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721 /*
2722 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2723 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2724 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2725 */
2726 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2727 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2728 || !pVM->hm.s.vmx.pRealModeTSS))
2729 {
2730 LogRelFunc(("Invalid real-on-v86 state.\n"));
2731 return VERR_INTERNAL_ERROR;
2732 }
2733
2734 /* Initialize these always, see hmR3InitFinalizeR0().*/
2735 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2736 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2737
2738 /* Setup the tagged-TLB flush handlers. */
2739 int rc = hmR0VmxSetupTaggedTlb(pVM);
2740 if (RT_FAILURE(rc))
2741 {
2742 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2743 return rc;
2744 }
2745
2746 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2747 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2748#if HC_ARCH_BITS == 64
2749 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2750 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2751 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2752 {
2753 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2754 }
2755#endif
2756
2757 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2758 RTCCUINTREG uHostCR4 = ASMGetCR4();
2759 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2760 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2761
2762 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2763 {
2764 PVMCPU pVCpu = &pVM->aCpus[i];
2765 AssertPtr(pVCpu);
2766 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2767
2768 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2769 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2770
2771 /* Set revision dword at the beginning of the VMCS structure. */
2772 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2773
2774 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2775 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2777 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2778
2779 /* Load this VMCS as the current VMCS. */
2780 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2781 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2782 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2783
2784 rc = hmR0VmxSetupPinCtls(pVCpu);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2786 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2787
2788 rc = hmR0VmxSetupProcCtls(pVCpu);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2790 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupMiscCtls(pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 rc = hmR0VmxInitXcptBitmap(pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800#if HC_ARCH_BITS == 32
2801 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804#endif
2805
2806 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2807 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2812
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2814 }
2815
2816 return VINF_SUCCESS;
2817}
2818
2819
2820/**
2821 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2822 * the VMCS.
2823 *
2824 * @returns VBox status code.
2825 */
2826static int hmR0VmxExportHostControlRegs(void)
2827{
2828 RTCCUINTREG uReg = ASMGetCR0();
2829 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2830 AssertRCReturn(rc, rc);
2831
2832 uReg = ASMGetCR3();
2833 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2834 AssertRCReturn(rc, rc);
2835
2836 uReg = ASMGetCR4();
2837 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2838 AssertRCReturn(rc, rc);
2839 return rc;
2840}
2841
2842
2843/**
2844 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2845 * the host-state area in the VMCS.
2846 *
2847 * @returns VBox status code.
2848 * @param pVCpu The cross context virtual CPU structure.
2849 */
2850static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2851{
2852#if HC_ARCH_BITS == 64
2853/**
2854 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2855 * requirements. See hmR0VmxExportHostSegmentRegs().
2856 */
2857# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2858 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2859 { \
2860 bool fValidSelector = true; \
2861 if ((selValue) & X86_SEL_LDT) \
2862 { \
2863 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2864 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2865 } \
2866 if (fValidSelector) \
2867 { \
2868 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2869 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2870 } \
2871 (selValue) = 0; \
2872 }
2873
2874 /*
2875 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2876 * should -not- save the messed up state without restoring the original host-state,
2877 * see @bugref{7240}.
2878 *
2879 * This apparently can happen (most likely the FPU changes), deal with it rather than
2880 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2881 */
2882 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2883 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2884 {
2885 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2886 pVCpu->idCpu));
2887 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2888 }
2889 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2890#else
2891 RT_NOREF(pVCpu);
2892#endif
2893
2894 /*
2895 * Host DS, ES, FS and GS segment registers.
2896 */
2897#if HC_ARCH_BITS == 64
2898 RTSEL uSelDS = ASMGetDS();
2899 RTSEL uSelES = ASMGetES();
2900 RTSEL uSelFS = ASMGetFS();
2901 RTSEL uSelGS = ASMGetGS();
2902#else
2903 RTSEL uSelDS = 0;
2904 RTSEL uSelES = 0;
2905 RTSEL uSelFS = 0;
2906 RTSEL uSelGS = 0;
2907#endif
2908
2909 /*
2910 * Host CS and SS segment registers.
2911 */
2912 RTSEL uSelCS = ASMGetCS();
2913 RTSEL uSelSS = ASMGetSS();
2914
2915 /*
2916 * Host TR segment register.
2917 */
2918 RTSEL uSelTR = ASMGetTR();
2919
2920#if HC_ARCH_BITS == 64
2921 /*
2922 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2923 * gain VM-entry and restore them before we get preempted.
2924 *
2925 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2926 */
2927 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2928 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2929 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2930 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2931# undef VMXLOCAL_ADJUST_HOST_SEG
2932#endif
2933
2934 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2935 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2936 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2937 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2938 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2939 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2940 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2941 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2942 Assert(uSelCS);
2943 Assert(uSelTR);
2944
2945 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2946#if 0
2947 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2948 Assert(uSelSS != 0);
2949#endif
2950
2951 /* Write these host selector fields into the host-state area in the VMCS. */
2952 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2953 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2954#if HC_ARCH_BITS == 64
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2956 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2959#else
2960 NOREF(uSelDS);
2961 NOREF(uSelES);
2962 NOREF(uSelFS);
2963 NOREF(uSelGS);
2964#endif
2965 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2966 AssertRCReturn(rc, rc);
2967
2968 /*
2969 * Host GDTR and IDTR.
2970 */
2971 RTGDTR Gdtr;
2972 RTIDTR Idtr;
2973 RT_ZERO(Gdtr);
2974 RT_ZERO(Idtr);
2975 ASMGetGDTR(&Gdtr);
2976 ASMGetIDTR(&Idtr);
2977 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2978 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2979 AssertRCReturn(rc, rc);
2980
2981#if HC_ARCH_BITS == 64
2982 /*
2983 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2984 * them to the maximum limit (0xffff) on every VM-exit.
2985 */
2986 if (Gdtr.cbGdt != 0xffff)
2987 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2988
2989 /*
2990 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2991 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2992 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2993 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2994 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2995 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2996 * at 0xffff on hosts where we are sure it won't cause trouble.
2997 */
2998# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2999 if (Idtr.cbIdt < 0x0fff)
3000# else
3001 if (Idtr.cbIdt != 0xffff)
3002# endif
3003 {
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3005 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3006 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3007 }
3008#endif
3009
3010 /*
3011 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3012 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3013 * RPL should be too in most cases.
3014 */
3015 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3016 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3017
3018 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3019#if HC_ARCH_BITS == 64
3020 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3021
3022 /*
3023 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3024 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3025 * restoration if the host has something else. Task switching is not supported in 64-bit
3026 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3027 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3028 *
3029 * [1] See Intel spec. 3.5 "System Descriptor Types".
3030 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3031 */
3032 PVM pVM = pVCpu->CTX_SUFF(pVM);
3033 Assert(pDesc->System.u4Type == 11);
3034 if ( pDesc->System.u16LimitLow != 0x67
3035 || pDesc->System.u4LimitHigh)
3036 {
3037 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3038 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3039 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3041 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3042 }
3043
3044 /*
3045 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3046 */
3047 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3048 {
3049 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3050 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3051 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3052 {
3053 /* The GDT is read-only but the writable GDT is available. */
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3055 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3056 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3057 AssertRCReturn(rc, rc);
3058 }
3059 }
3060#else
3061 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3062#endif
3063 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3064 AssertRCReturn(rc, rc);
3065
3066 /*
3067 * Host FS base and GS base.
3068 */
3069#if HC_ARCH_BITS == 64
3070 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3071 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3072 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3073 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3074 AssertRCReturn(rc, rc);
3075
3076 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3077 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3078 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3079 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3080 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3081#endif
3082 return VINF_SUCCESS;
3083}
3084
3085
3086/**
3087 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3088 * host-state area of the VMCS.
3089 *
3090 * Theses MSRs will be automatically restored on the host after every successful
3091 * VM-exit.
3092 *
3093 * @returns VBox status code.
3094 * @param pVCpu The cross context virtual CPU structure.
3095 *
3096 * @remarks No-long-jump zone!!!
3097 */
3098static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3099{
3100 AssertPtr(pVCpu);
3101 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3102
3103 /*
3104 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3105 * rather than swapping them on every VM-entry.
3106 */
3107 hmR0VmxLazySaveHostMsrs(pVCpu);
3108
3109 /*
3110 * Host Sysenter MSRs.
3111 */
3112 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3113#if HC_ARCH_BITS == 32
3114 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3115 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3116#else
3117 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3118 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3119#endif
3120 AssertRCReturn(rc, rc);
3121
3122 /*
3123 * Host EFER MSR.
3124 *
3125 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3126 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3127 */
3128 PVM pVM = pVCpu->CTX_SUFF(pVM);
3129 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3130 {
3131 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3132 AssertRCReturn(rc, rc);
3133 }
3134
3135 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3136
3137 return VINF_SUCCESS;
3138}
3139
3140
3141/**
3142 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3143 *
3144 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3145 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3146 * hmR0VMxExportGuestEntryCtls().
3147 *
3148 * @returns true if we need to load guest EFER, false otherwise.
3149 * @param pVCpu The cross context virtual CPU structure.
3150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3151 * out-of-sync. Make sure to update the required fields
3152 * before using them.
3153 *
3154 * @remarks Requires EFER, CR4.
3155 * @remarks No-long-jump zone!!!
3156 */
3157static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3158{
3159#ifdef HMVMX_ALWAYS_SWAP_EFER
3160 return true;
3161#endif
3162
3163#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3164 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3165 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3166 return false;
3167#endif
3168
3169 PVM pVM = pVCpu->CTX_SUFF(pVM);
3170 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3171 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3172
3173 /*
3174 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3175 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3176 */
3177 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3178 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3179 {
3180 return true;
3181 }
3182
3183 /*
3184 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3185 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3186 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3187 */
3188 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3189 && (pMixedCtx->cr0 & X86_CR0_PG)
3190 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3191 {
3192 /* Assert that host is PAE capable. */
3193 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3194 return true;
3195 }
3196
3197 return false;
3198}
3199
3200
3201/**
3202 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3203 *
3204 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3205 * see Intel spec. 24.8.1 "VM-entry controls".
3206 *
3207 * @returns VBox status code.
3208 * @param pVCpu The cross context virtual CPU structure.
3209 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3210 * out-of-sync. Make sure to update the required fields
3211 * before using them.
3212 *
3213 * @remarks Requires EFER.
3214 * @remarks No-long-jump zone!!!
3215 */
3216static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3217{
3218 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3219 {
3220 PVM pVM = pVCpu->CTX_SUFF(pVM);
3221 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3222 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3223
3224 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3225 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3226
3227 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3228 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3229 {
3230 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3231 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3232 }
3233 else
3234 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3235
3236 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3237 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3238 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3239 {
3240 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3241 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3242 }
3243
3244 /*
3245 * The following should -not- be set (since we're not in SMM mode):
3246 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3247 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3248 */
3249
3250 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3251 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3252
3253 if ((fVal & fZap) != fVal)
3254 {
3255 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3256 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3257 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3259 }
3260
3261 /* Commit it to the VMCS and update our cache. */
3262 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3263 {
3264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3265 AssertRCReturn(rc, rc);
3266 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3267 }
3268
3269 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3270 }
3271 return VINF_SUCCESS;
3272}
3273
3274
3275/**
3276 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3277 *
3278 * @returns VBox status code.
3279 * @param pVCpu The cross context virtual CPU structure.
3280 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3281 * out-of-sync. Make sure to update the required fields
3282 * before using them.
3283 *
3284 * @remarks Requires EFER.
3285 */
3286static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3287{
3288 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3289 {
3290 PVM pVM = pVCpu->CTX_SUFF(pVM);
3291 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3292 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3293
3294 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3295 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3296
3297 /*
3298 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3299 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3300 * hmR0VmxExportHostMsrs().
3301 */
3302#if HC_ARCH_BITS == 64
3303 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3304 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3305#else
3306 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3307 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3308 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3309 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3310 {
3311 /* The switcher returns to long mode, EFER is managed by the switcher. */
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314 }
3315 else
3316 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3317#endif
3318
3319 /* If the newer VMCS fields for managing EFER exists, use it. */
3320 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322 {
3323 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3324 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3325 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3326 }
3327
3328 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3329 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3330
3331 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3332 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3333 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3334
3335 /* Enable saving of the VMX preemption timer value on VM-exit. */
3336 if ( pVM->hm.s.vmx.fUsePreemptTimer
3337 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3338 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3339
3340 if ((fVal & fZap) != fVal)
3341 {
3342 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3343 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3344 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3345 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3346 }
3347
3348 /* Commit it to the VMCS and update our cache. */
3349 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3350 {
3351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3352 AssertRCReturn(rc, rc);
3353 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3354 }
3355
3356 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3357 }
3358 return VINF_SUCCESS;
3359}
3360
3361
3362/**
3363 * Sets the TPR threshold in the VMCS.
3364 *
3365 * @returns VBox status code.
3366 * @param pVCpu The cross context virtual CPU structure.
3367 * @param u32TprThreshold The TPR threshold (task-priority class only).
3368 */
3369DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3370{
3371 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3372 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3373 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3374}
3375
3376
3377/**
3378 * Exports the guest APIC TPR state into the VMCS.
3379 *
3380 * @returns VBox status code.
3381 * @param pVCpu The cross context virtual CPU structure.
3382 *
3383 * @remarks No-long-jump zone!!!
3384 */
3385static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3386{
3387 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3388 {
3389 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3390 && APICIsEnabled(pVCpu))
3391 {
3392 /*
3393 * Setup TPR shadowing.
3394 */
3395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3396 {
3397 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3398
3399 bool fPendingIntr = false;
3400 uint8_t u8Tpr = 0;
3401 uint8_t u8PendingIntr = 0;
3402 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3403 AssertRCReturn(rc, rc);
3404
3405 /*
3406 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3407 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3408 * priority of the pending interrupt so we can deliver the interrupt. If there
3409 * are no interrupts pending, set threshold to 0 to not cause any
3410 * TPR-below-threshold VM-exits.
3411 */
3412 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3413 uint32_t u32TprThreshold = 0;
3414 if (fPendingIntr)
3415 {
3416 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3417 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3418 const uint8_t u8TprPriority = u8Tpr >> 4;
3419 if (u8PendingPriority <= u8TprPriority)
3420 u32TprThreshold = u8PendingPriority;
3421 }
3422
3423 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3424 AssertRCReturn(rc, rc);
3425 }
3426 }
3427 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3428 }
3429 return VINF_SUCCESS;
3430}
3431
3432
3433/**
3434 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3435 *
3436 * @returns Guest's interruptibility-state.
3437 * @param pVCpu The cross context virtual CPU structure.
3438 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3439 * out-of-sync. Make sure to update the required fields
3440 * before using them.
3441 *
3442 * @remarks No-long-jump zone!!!
3443 */
3444static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3445{
3446 /*
3447 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3448 */
3449 uint32_t fIntrState = 0;
3450 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3451 {
3452 /* If inhibition is active, RIP & RFLAGS should've been accessed
3453 (i.e. read previously from the VMCS or from ring-3). */
3454#ifdef VBOX_STRICT
3455 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3456 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3457#endif
3458 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3459 {
3460 if (pMixedCtx->eflags.Bits.u1IF)
3461 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3462 else
3463 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3464 }
3465 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466 {
3467 /*
3468 * We can clear the inhibit force flag as even if we go back to the recompiler
3469 * without executing guest code in VT-x, the flag's condition to be cleared is
3470 * met and thus the cleared state is correct.
3471 */
3472 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3473 }
3474 }
3475
3476 /*
3477 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3478 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3479 * setting this would block host-NMIs and IRET will not clear the blocking.
3480 *
3481 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3482 */
3483 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3484 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3485 {
3486 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3487 }
3488
3489 return fIntrState;
3490}
3491
3492
3493/**
3494 * Exports the guest's interruptibility-state into the guest-state area in the
3495 * VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 * @param fIntrState The interruptibility-state to set.
3500 */
3501static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3502{
3503 NOREF(pVCpu);
3504 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3505 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3506 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3507}
3508
3509
3510/**
3511 * Exports the exception intercepts required for guest execution in the VMCS.
3512 *
3513 * @returns VBox status code.
3514 * @param pVCpu The cross context virtual CPU structure.
3515 *
3516 * @remarks No-long-jump zone!!!
3517 */
3518static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3519{
3520 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3521 {
3522 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3523 if (pVCpu->hm.s.fGIMTrapXcptUD)
3524 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3525#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3526 else
3527 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3528#endif
3529
3530 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3531 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3532
3533 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3534 AssertRCReturn(rc, rc);
3535
3536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3537 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3538 }
3539 return VINF_SUCCESS;
3540}
3541
3542
3543/**
3544 * Exports the guest's RIP into the guest-state area in the VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3549 * out-of-sync. Make sure to update the required fields
3550 * before using them.
3551 *
3552 * @remarks No-long-jump zone!!!
3553 */
3554static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3555{
3556 int rc = VINF_SUCCESS;
3557 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3558 {
3559 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3560 AssertRCReturn(rc, rc);
3561
3562 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3564 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3565 else
3566 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3567
3568 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3569 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3570 }
3571 return rc;
3572}
3573
3574
3575/**
3576 * Exports the guest's RSP into the guest-state area in the VMCS.
3577 *
3578 * @returns VBox status code.
3579 * @param pVCpu The cross context virtual CPU structure.
3580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3581 * out-of-sync. Make sure to update the required fields
3582 * before using them.
3583 *
3584 * @remarks No-long-jump zone!!!
3585 */
3586static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3587{
3588 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3589 {
3590 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3591 AssertRCReturn(rc, rc);
3592
3593 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3594 }
3595 return VINF_SUCCESS;
3596}
3597
3598
3599/**
3600 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3601 *
3602 * @returns VBox status code.
3603 * @param pVCpu The cross context virtual CPU structure.
3604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3605 * out-of-sync. Make sure to update the required fields
3606 * before using them.
3607 *
3608 * @remarks No-long-jump zone!!!
3609 */
3610static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3611{
3612 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613 {
3614 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3615 Let us assert it as such and use 32-bit VMWRITE. */
3616 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3617 X86EFLAGS fEFlags = pMixedCtx->eflags;
3618 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3619 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3620
3621 /*
3622 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3623 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3624 * can run the real-mode guest code under Virtual 8086 mode.
3625 */
3626 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3627 {
3628 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3629 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3630 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3631 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3632 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3633 }
3634
3635 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3636 AssertRCReturn(rc, rc);
3637
3638 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3639 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3640 }
3641 return VINF_SUCCESS;
3642}
3643
3644
3645/**
3646 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3647 *
3648 * The guest FPU state is always pre-loaded hence we don't need to bother about
3649 * sharing FPU related CR0 bits between the guest and host.
3650 *
3651 * @returns VBox status code.
3652 * @param pVCpu The cross context virtual CPU structure.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3660{
3661 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3662 {
3663 PVM pVM = pVCpu->CTX_SUFF(pVM);
3664 Assert(!RT_HI_U32(pMixedCtx->cr0));
3665 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3666 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3667
3668 /*
3669 * Setup VT-x's view of the guest CR0.
3670 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3671 */
3672 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3673 if (pVM->hm.s.fNestedPaging)
3674 {
3675 if (CPUMIsGuestPagingEnabled(pVCpu))
3676 {
3677 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3678 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3679 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3680 }
3681 else
3682 {
3683 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3684 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3685 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3686 }
3687
3688 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3689 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3690 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692 else
3693 {
3694 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3695 u32GuestCr0 |= X86_CR0_WP;
3696 }
3697
3698 /*
3699 * Guest FPU bits.
3700 *
3701 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3702 * using CR0.TS.
3703 *
3704 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3705 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3706 */
3707 u32GuestCr0 |= X86_CR0_NE;
3708
3709 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3710 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3711
3712 /*
3713 * Update exception intercepts.
3714 */
3715 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3716 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3717 {
3718 Assert(PDMVmmDevHeapIsEnabled(pVM));
3719 Assert(pVM->hm.s.vmx.pRealModeTSS);
3720 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3721 }
3722 else
3723 {
3724 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3725 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3726 if (fInterceptMF)
3727 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3728 }
3729
3730 /* Additional intercepts for debugging, define these yourself explicitly. */
3731#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3732 uXcptBitmap |= 0
3733 | RT_BIT(X86_XCPT_BP)
3734 | RT_BIT(X86_XCPT_DE)
3735 | RT_BIT(X86_XCPT_NM)
3736 | RT_BIT(X86_XCPT_TS)
3737 | RT_BIT(X86_XCPT_UD)
3738 | RT_BIT(X86_XCPT_NP)
3739 | RT_BIT(X86_XCPT_SS)
3740 | RT_BIT(X86_XCPT_GP)
3741 | RT_BIT(X86_XCPT_PF)
3742 | RT_BIT(X86_XCPT_MF)
3743 ;
3744#elif defined(HMVMX_ALWAYS_TRAP_PF)
3745 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3746#endif
3747 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3748 {
3749 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3751 }
3752 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 AssertRCReturn(rc, rc);
3798
3799 /* Update our caches. */
3800 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3801 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3802
3803 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3804
3805 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3806 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3807 }
3808
3809 return VINF_SUCCESS;
3810}
3811
3812
3813/**
3814 * Exports the guest control registers (CR3, CR4) into the guest-state area
3815 * in the VMCS.
3816 *
3817 * @returns VBox strict status code.
3818 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3819 * without unrestricted guest access and the VMMDev is not presently
3820 * mapped (e.g. EFI32).
3821 *
3822 * @param pVCpu The cross context virtual CPU structure.
3823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3824 * out-of-sync. Make sure to update the required fields
3825 * before using them.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3845 if (pVM->hm.s.fNestedPaging)
3846 {
3847 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3848
3849 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3850 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3851 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3853
3854 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3855 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3856 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3857
3858 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3859 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3860 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3861 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3863 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3864 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3865
3866 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3867 AssertRCReturn(rc, rc);
3868
3869 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3870 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3871 {
3872 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3873 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3874 {
3875 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3876 AssertRCReturn(rc, rc);
3877 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3878 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3881 AssertRCReturn(rc, rc);
3882 }
3883
3884 /*
3885 * The guest's view of its CR3 is unblemished with Nested Paging when the
3886 * guest is using paging or we have unrestricted guest execution to handle
3887 * the guest when it's not using paging.
3888 */
3889 GCPhysGuestCR3 = pMixedCtx->cr3;
3890 }
3891 else
3892 {
3893 /*
3894 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3895 * thinks it accesses physical memory directly, we use our identity-mapped
3896 * page table to map guest-linear to guest-physical addresses. EPT takes care
3897 * of translating it to host-physical addresses.
3898 */
3899 RTGCPHYS GCPhys;
3900 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3901
3902 /* We obtain it here every time as the guest could have relocated this PCI region. */
3903 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3904 if (RT_SUCCESS(rc))
3905 { /* likely */ }
3906 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3907 {
3908 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3909 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3910 }
3911 else
3912 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3913
3914 GCPhysGuestCR3 = GCPhys;
3915 }
3916
3917 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3918 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3919 AssertRCReturn(rc, rc);
3920 }
3921 else
3922 {
3923 /* Non-nested paging case, just use the hypervisor's CR3. */
3924 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3925
3926 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3927 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3928 AssertRCReturn(rc, rc);
3929 }
3930
3931 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3932 }
3933
3934 /*
3935 * Guest CR4.
3936 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3937 */
3938 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3939 {
3940 Assert(!RT_HI_U32(pMixedCtx->cr4));
3941 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3942 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3943
3944 /*
3945 * Setup VT-x's view of the guest CR4.
3946 *
3947 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3948 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3949 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3950 *
3951 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3952 */
3953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3954 {
3955 Assert(pVM->hm.s.vmx.pRealModeTSS);
3956 Assert(PDMVmmDevHeapIsEnabled(pVM));
3957 u32GuestCr4 &= ~X86_CR4_VME;
3958 }
3959
3960 if (pVM->hm.s.fNestedPaging)
3961 {
3962 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3963 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3964 {
3965 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3966 u32GuestCr4 |= X86_CR4_PSE;
3967 /* Our identity mapping is a 32-bit page directory. */
3968 u32GuestCr4 &= ~X86_CR4_PAE;
3969 }
3970 /* else use guest CR4.*/
3971 }
3972 else
3973 {
3974 /*
3975 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3976 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3977 */
3978 switch (pVCpu->hm.s.enmShadowMode)
3979 {
3980 case PGMMODE_REAL: /* Real-mode. */
3981 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3982 case PGMMODE_32_BIT: /* 32-bit paging. */
3983 {
3984 u32GuestCr4 &= ~X86_CR4_PAE;
3985 break;
3986 }
3987
3988 case PGMMODE_PAE: /* PAE paging. */
3989 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3990 {
3991 u32GuestCr4 |= X86_CR4_PAE;
3992 break;
3993 }
3994
3995 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3996 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3997#ifdef VBOX_ENABLE_64_BITS_GUESTS
3998 break;
3999#endif
4000 default:
4001 AssertFailed();
4002 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4003 }
4004 }
4005
4006 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4007 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4008 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4009 u32GuestCr4 |= fSetCr4;
4010 u32GuestCr4 &= fZapCr4;
4011
4012 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4013 that would cause a VM-exit. */
4014 uint32_t u32Cr4Mask = X86_CR4_VME
4015 | X86_CR4_PAE
4016 | X86_CR4_PGE
4017 | X86_CR4_PSE
4018 | X86_CR4_VMXE;
4019 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4020 u32Cr4Mask |= X86_CR4_OSXSAVE;
4021 if (pVM->cpum.ro.GuestFeatures.fPcid)
4022 u32Cr4Mask |= X86_CR4_PCIDE;
4023
4024 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4025 into the VMCS and update our cache. */
4026 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4027 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4028 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4029 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4030 AssertRCReturn(rc, rc);
4031 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4032
4033 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4034 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4035
4036 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4037
4038 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4039 fZapCr4));
4040 }
4041 return rc;
4042}
4043
4044
4045/**
4046 * Exports the guest debug registers into the guest-state area in the VMCS.
4047 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4048 *
4049 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4050 *
4051 * @returns VBox status code.
4052 * @param pVCpu The cross context virtual CPU structure.
4053 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4054 * out-of-sync. Make sure to update the required fields
4055 * before using them.
4056 *
4057 * @remarks No-long-jump zone!!!
4058 */
4059static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4060{
4061 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4062
4063#ifdef VBOX_STRICT
4064 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4065 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4066 {
4067 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4068 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4069 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4070 }
4071#endif
4072
4073 bool fSteppingDB = false;
4074 bool fInterceptMovDRx = false;
4075 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4076 if (pVCpu->hm.s.fSingleInstruction)
4077 {
4078 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4079 PVM pVM = pVCpu->CTX_SUFF(pVM);
4080 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4081 {
4082 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4083 Assert(fSteppingDB == false);
4084 }
4085 else
4086 {
4087 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4088 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4089 pVCpu->hm.s.fClearTrapFlag = true;
4090 fSteppingDB = true;
4091 }
4092 }
4093
4094 uint32_t u32GuestDr7;
4095 if ( fSteppingDB
4096 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4097 {
4098 /*
4099 * Use the combined guest and host DRx values found in the hypervisor register set
4100 * because the debugger has breakpoints active or someone is single stepping on the
4101 * host side without a monitor trap flag.
4102 *
4103 * Note! DBGF expects a clean DR6 state before executing guest code.
4104 */
4105#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4106 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4107 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4108 {
4109 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4110 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4111 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4112 }
4113 else
4114#endif
4115 if (!CPUMIsHyperDebugStateActive(pVCpu))
4116 {
4117 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4118 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4119 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4120 }
4121
4122 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4123 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4124 pVCpu->hm.s.fUsingHyperDR7 = true;
4125 fInterceptMovDRx = true;
4126 }
4127 else
4128 {
4129 /*
4130 * If the guest has enabled debug registers, we need to load them prior to
4131 * executing guest code so they'll trigger at the right time.
4132 */
4133 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4134 {
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4142 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4143 }
4144 else
4145#endif
4146 if (!CPUMIsGuestDebugStateActive(pVCpu))
4147 {
4148 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4149 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4150 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4151 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4152 }
4153 Assert(!fInterceptMovDRx);
4154 }
4155 /*
4156 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4157 * must intercept #DB in order to maintain a correct DR6 guest value, and
4158 * because we need to intercept it to prevent nested #DBs from hanging the
4159 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4160 */
4161#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4162 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4163 && !CPUMIsGuestDebugStateActive(pVCpu))
4164#else
4165 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4166#endif
4167 {
4168 fInterceptMovDRx = true;
4169 }
4170
4171 /* Update DR7 with the actual guest value. */
4172 u32GuestDr7 = pMixedCtx->dr[7];
4173 pVCpu->hm.s.fUsingHyperDR7 = false;
4174 }
4175
4176 if (fInterceptMovDRx)
4177 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4178 else
4179 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4180
4181 /*
4182 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4183 * monitor-trap flag and update our cache.
4184 */
4185 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4186 {
4187 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4188 AssertRCReturn(rc2, rc2);
4189 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4190 }
4191
4192 /*
4193 * Update guest DR7.
4194 */
4195 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4196 AssertRCReturn(rc, rc);
4197
4198 return VINF_SUCCESS;
4199}
4200
4201
4202#ifdef VBOX_STRICT
4203/**
4204 * Strict function to validate segment registers.
4205 *
4206 * @param pVCpu The cross context virtual CPU structure.
4207 * @param pCtx Pointer to the guest-CPU context.
4208 *
4209 * @remarks Will import guest CR0 on strict builds during validation of
4210 * segments.
4211 */
4212static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4213{
4214 /*
4215 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4216 *
4217 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4218 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4219 * and doesn't change the guest-context value.
4220 */
4221 PVM pVM = pVCpu->CTX_SUFF(pVM);
4222 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4223 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4224 && ( !CPUMIsGuestInRealModeEx(pCtx)
4225 && !CPUMIsGuestInV86ModeEx(pCtx)))
4226 {
4227 /* Protected mode checks */
4228 /* CS */
4229 Assert(pCtx->cs.Attr.n.u1Present);
4230 Assert(!(pCtx->cs.Attr.u & 0xf00));
4231 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4232 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4233 || !(pCtx->cs.Attr.n.u1Granularity));
4234 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4235 || (pCtx->cs.Attr.n.u1Granularity));
4236 /* CS cannot be loaded with NULL in protected mode. */
4237 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4238 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4239 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4240 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4241 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4242 else
4243 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4244 /* SS */
4245 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4246 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4247 if ( !(pCtx->cr0 & X86_CR0_PE)
4248 || pCtx->cs.Attr.n.u4Type == 3)
4249 {
4250 Assert(!pCtx->ss.Attr.n.u2Dpl);
4251 }
4252 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4253 {
4254 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4255 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4256 Assert(pCtx->ss.Attr.n.u1Present);
4257 Assert(!(pCtx->ss.Attr.u & 0xf00));
4258 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4259 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4260 || !(pCtx->ss.Attr.n.u1Granularity));
4261 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4262 || (pCtx->ss.Attr.n.u1Granularity));
4263 }
4264 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4265 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4266 {
4267 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4268 Assert(pCtx->ds.Attr.n.u1Present);
4269 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4270 Assert(!(pCtx->ds.Attr.u & 0xf00));
4271 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4272 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4273 || !(pCtx->ds.Attr.n.u1Granularity));
4274 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4275 || (pCtx->ds.Attr.n.u1Granularity));
4276 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4277 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4278 }
4279 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4280 {
4281 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4282 Assert(pCtx->es.Attr.n.u1Present);
4283 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4284 Assert(!(pCtx->es.Attr.u & 0xf00));
4285 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4286 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4287 || !(pCtx->es.Attr.n.u1Granularity));
4288 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4289 || (pCtx->es.Attr.n.u1Granularity));
4290 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4291 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4292 }
4293 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->fs.Attr.n.u1Present);
4297 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->fs.Attr.u & 0xf00));
4299 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->fs.Attr.n.u1Granularity));
4302 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4303 || (pCtx->fs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->gs.Attr.n.u1Present);
4311 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->gs.Attr.u & 0xf00));
4313 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->gs.Attr.n.u1Granularity));
4316 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4317 || (pCtx->gs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 /* 64-bit capable CPUs. */
4322# if HC_ARCH_BITS == 64
4323 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4324 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4325 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4326 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4327# endif
4328 }
4329 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4330 || ( CPUMIsGuestInRealModeEx(pCtx)
4331 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4332 {
4333 /* Real and v86 mode checks. */
4334 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4335 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4336 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4337 {
4338 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4339 }
4340 else
4341 {
4342 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4343 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4344 }
4345
4346 /* CS */
4347 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4348 Assert(pCtx->cs.u32Limit == 0xffff);
4349 Assert(u32CSAttr == 0xf3);
4350 /* SS */
4351 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4352 Assert(pCtx->ss.u32Limit == 0xffff);
4353 Assert(u32SSAttr == 0xf3);
4354 /* DS */
4355 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4356 Assert(pCtx->ds.u32Limit == 0xffff);
4357 Assert(u32DSAttr == 0xf3);
4358 /* ES */
4359 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4360 Assert(pCtx->es.u32Limit == 0xffff);
4361 Assert(u32ESAttr == 0xf3);
4362 /* FS */
4363 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4364 Assert(pCtx->fs.u32Limit == 0xffff);
4365 Assert(u32FSAttr == 0xf3);
4366 /* GS */
4367 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4368 Assert(pCtx->gs.u32Limit == 0xffff);
4369 Assert(u32GSAttr == 0xf3);
4370 /* 64-bit capable CPUs. */
4371# if HC_ARCH_BITS == 64
4372 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4373 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4374 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4375 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4376# endif
4377 }
4378}
4379#endif /* VBOX_STRICT */
4380
4381
4382/**
4383 * Writes a guest segment register into the guest-state area in the VMCS.
4384 *
4385 * @returns VBox status code.
4386 * @param pVCpu The cross context virtual CPU structure.
4387 * @param idxSel Index of the selector in the VMCS.
4388 * @param idxLimit Index of the segment limit in the VMCS.
4389 * @param idxBase Index of the segment base in the VMCS.
4390 * @param idxAccess Index of the access rights of the segment in the VMCS.
4391 * @param pSelReg Pointer to the segment selector.
4392 *
4393 * @remarks No-long-jump zone!!!
4394 */
4395static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4396 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4397{
4398 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4399 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4400 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4401 AssertRCReturn(rc, rc);
4402
4403 uint32_t u32Access = pSelReg->Attr.u;
4404 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4405 {
4406 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4407 u32Access = 0xf3;
4408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4409 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4410 }
4411 else
4412 {
4413 /*
4414 * The way to differentiate between whether this is really a null selector or was just
4415 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4416 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4417 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4418 * NULL selectors loaded in protected-mode have their attribute as 0.
4419 */
4420 if (!u32Access)
4421 u32Access = X86DESCATTR_UNUSABLE;
4422 }
4423
4424 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4425 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4426 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4427
4428 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4429 AssertRCReturn(rc, rc);
4430 return rc;
4431}
4432
4433
4434/**
4435 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4436 * into the guest-state area in the VMCS.
4437 *
4438 * @returns VBox status code.
4439 * @param pVCpu The cross context virtual CPU structure.
4440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4441 * out-of-sync. Make sure to update the required fields
4442 * before using them.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452
4453 /*
4454 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4455 */
4456 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4457 {
4458 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4459 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4460 {
4461 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4462 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4463 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4464 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4465 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4466 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4467 }
4468
4469#ifdef VBOX_WITH_REM
4470 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4471 {
4472 Assert(pVM->hm.s.vmx.pRealModeTSS);
4473 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4474 if ( pVCpu->hm.s.vmx.fWasInRealMode
4475 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4476 {
4477 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4478 in real-mode (e.g. OpenBSD 4.0) */
4479 REMFlushTBs(pVM);
4480 Log4Func(("Switch to protected mode detected!\n"));
4481 pVCpu->hm.s.vmx.fWasInRealMode = false;
4482 }
4483 }
4484#endif
4485 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4486 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4487 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4488 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4489 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4490 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4491 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4492 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4493 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4494 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4495 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4496 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4497 AssertRCReturn(rc, rc);
4498
4499#ifdef VBOX_STRICT
4500 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4501#endif
4502
4503 /* Update the exit history entry with the correct CS.BASE + RIP. */
4504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4505 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4506
4507 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4508 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4509 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4510 }
4511
4512 /*
4513 * Guest TR.
4514 */
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4516 {
4517 /*
4518 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4519 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4520 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4521 */
4522 uint16_t u16Sel = 0;
4523 uint32_t u32Limit = 0;
4524 uint64_t u64Base = 0;
4525 uint32_t u32AccessRights = 0;
4526
4527 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4528 {
4529 u16Sel = pMixedCtx->tr.Sel;
4530 u32Limit = pMixedCtx->tr.u32Limit;
4531 u64Base = pMixedCtx->tr.u64Base;
4532 u32AccessRights = pMixedCtx->tr.Attr.u;
4533 }
4534 else
4535 {
4536 Assert(pVM->hm.s.vmx.pRealModeTSS);
4537 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4538
4539 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4540 RTGCPHYS GCPhys;
4541 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4542 AssertRCReturn(rc, rc);
4543
4544 X86DESCATTR DescAttr;
4545 DescAttr.u = 0;
4546 DescAttr.n.u1Present = 1;
4547 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4548
4549 u16Sel = 0;
4550 u32Limit = HM_VTX_TSS_SIZE;
4551 u64Base = GCPhys; /* in real-mode phys = virt. */
4552 u32AccessRights = DescAttr.u;
4553 }
4554
4555 /* Validate. */
4556 Assert(!(u16Sel & RT_BIT(2)));
4557 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4558 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4559 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4560 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4561 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4562 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4563 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4564 Assert( (u32Limit & 0xfff) == 0xfff
4565 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4566 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4567 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4568
4569 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4570 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4571 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4572 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4573 AssertRCReturn(rc, rc);
4574
4575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4576 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4577 }
4578
4579 /*
4580 * Guest GDTR.
4581 */
4582 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4583 {
4584 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4585 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4586 AssertRCReturn(rc, rc);
4587
4588 /* Validate. */
4589 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4590
4591 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4592 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4593 }
4594
4595 /*
4596 * Guest LDTR.
4597 */
4598 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4599 {
4600 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4601 uint32_t u32Access = 0;
4602 if (!pMixedCtx->ldtr.Attr.u)
4603 u32Access = X86DESCATTR_UNUSABLE;
4604 else
4605 u32Access = pMixedCtx->ldtr.Attr.u;
4606
4607 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4609 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4610 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4611 AssertRCReturn(rc, rc);
4612
4613 /* Validate. */
4614 if (!(u32Access & X86DESCATTR_UNUSABLE))
4615 {
4616 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4617 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4618 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4619 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4620 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4621 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4622 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4623 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4624 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4625 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4626 }
4627
4628 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4629 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4630 }
4631
4632 /*
4633 * Guest IDTR.
4634 */
4635 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4636 {
4637 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4638 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4639 AssertRCReturn(rc, rc);
4640
4641 /* Validate. */
4642 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4643
4644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4645 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4646 }
4647
4648 return VINF_SUCCESS;
4649}
4650
4651
4652/**
4653 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4654 * areas.
4655 *
4656 * These MSRs will automatically be loaded to the host CPU on every successful
4657 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4658 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4659 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4660 *
4661 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4662 *
4663 * @returns VBox status code.
4664 * @param pVCpu The cross context virtual CPU structure.
4665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4666 * out-of-sync. Make sure to update the required fields
4667 * before using them.
4668 *
4669 * @remarks No-long-jump zone!!!
4670 */
4671static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4672{
4673 AssertPtr(pVCpu);
4674 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4675
4676 /*
4677 * MSRs that we use the auto-load/store MSR area in the VMCS.
4678 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4679 */
4680 PVM pVM = pVCpu->CTX_SUFF(pVM);
4681 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4682 {
4683 if (pVM->hm.s.fAllow64BitGuests)
4684 {
4685#if HC_ARCH_BITS == 32
4686 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4687 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4688 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4689 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4690 AssertRCReturn(rc, rc);
4691# ifdef LOG_ENABLED
4692 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4693 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4694 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4695# endif
4696#endif
4697 }
4698 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4699 }
4700
4701 /*
4702 * Guest Sysenter MSRs.
4703 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4704 * VM-exits on WRMSRs for these MSRs.
4705 */
4706 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4707 {
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4709 {
4710 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4711 AssertRCReturn(rc, rc);
4712 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4713 }
4714
4715 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4716 {
4717 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4718 AssertRCReturn(rc, rc);
4719 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4720 }
4721
4722 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4723 {
4724 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4725 AssertRCReturn(rc, rc);
4726 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4727 }
4728 }
4729
4730 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4731 {
4732 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4733 {
4734 /*
4735 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4736 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4737 */
4738 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4739 {
4740 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4741 AssertRCReturn(rc,rc);
4742 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4743 }
4744 else
4745 {
4746 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4747 NULL /* pfAddedAndUpdated */);
4748 AssertRCReturn(rc, rc);
4749
4750 /* We need to intercept reads too, see @bugref{7386#c16}. */
4751 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4752 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4753 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4754 pVCpu->hm.s.vmx.cMsrs));
4755 }
4756 }
4757 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4758 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4760 }
4761
4762 return VINF_SUCCESS;
4763}
4764
4765
4766#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4767/**
4768 * Check if guest state allows safe use of 32-bit switcher again.
4769 *
4770 * Segment bases and protected mode structures must be 32-bit addressable
4771 * because the 32-bit switcher will ignore high dword when writing these VMCS
4772 * fields. See @bugref{8432} for details.
4773 *
4774 * @returns true if safe, false if must continue to use the 64-bit switcher.
4775 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4776 * out-of-sync. Make sure to update the required fields
4777 * before using them.
4778 *
4779 * @remarks No-long-jump zone!!!
4780 */
4781static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4782{
4783 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4784 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4785 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4786 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4787 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4788 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4789 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4790 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4791 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4792 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4793
4794 /* All good, bases are 32-bit. */
4795 return true;
4796}
4797#endif
4798
4799
4800/**
4801 * Selects up the appropriate function to run guest code.
4802 *
4803 * @returns VBox status code.
4804 * @param pVCpu The cross context virtual CPU structure.
4805 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4806 * out-of-sync. Make sure to update the required fields
4807 * before using them.
4808 *
4809 * @remarks No-long-jump zone!!!
4810 */
4811static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4812{
4813 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4814 {
4815#ifndef VBOX_ENABLE_64_BITS_GUESTS
4816 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4817#endif
4818 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4819#if HC_ARCH_BITS == 32
4820 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4821 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4822 {
4823#ifdef VBOX_STRICT
4824 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4825 {
4826 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4827 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4828 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4829 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4830 | HM_CHANGED_VMX_ENTRY_CTLS
4831 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4832 }
4833#endif
4834 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4835
4836 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4837 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4838 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4839 Log4Func(("Selected 64-bit switcher\n"));
4840 }
4841#else
4842 /* 64-bit host. */
4843 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4844#endif
4845 }
4846 else
4847 {
4848 /* Guest is not in long mode, use the 32-bit handler. */
4849#if HC_ARCH_BITS == 32
4850 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4851 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4852 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4853 {
4854# ifdef VBOX_STRICT
4855 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4856 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4857 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4858 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4859 | HM_CHANGED_VMX_ENTRY_CTLS
4860 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4861# endif
4862 }
4863# ifdef VBOX_ENABLE_64_BITS_GUESTS
4864 /*
4865 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4866 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4867 * switcher flag because now we know the guest is in a sane state where it's safe
4868 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4869 * the much faster 32-bit switcher again.
4870 */
4871 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4872 {
4873 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4874 Log4Func(("Selected 32-bit switcher\n"));
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876 }
4877 else
4878 {
4879 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4880 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4881 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4882 {
4883 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4884 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4885 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4886 | HM_CHANGED_VMX_ENTRY_CTLS
4887 | HM_CHANGED_VMX_EXIT_CTLS
4888 | HM_CHANGED_HOST_CONTEXT);
4889 Log4Func(("Selected 32-bit switcher (safe)\n"));
4890 }
4891 }
4892# else
4893 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4894# endif
4895#else
4896 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4897#endif
4898 }
4899 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4900 return VINF_SUCCESS;
4901}
4902
4903
4904/**
4905 * Wrapper for running the guest code in VT-x.
4906 *
4907 * @returns VBox status code, no informational status codes.
4908 * @param pVCpu The cross context virtual CPU structure.
4909 * @param pCtx Pointer to the guest-CPU context.
4910 *
4911 * @remarks No-long-jump zone!!!
4912 */
4913DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4914{
4915 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4916 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4917
4918 /*
4919 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4920 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4921 * callee-saved and thus the need for this XMM wrapper.
4922 *
4923 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4924 */
4925 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4926 /** @todo Add stats for resume vs launch. */
4927 PVM pVM = pVCpu->CTX_SUFF(pVM);
4928#ifdef VBOX_WITH_KERNEL_USING_XMM
4929 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4930#else
4931 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4932#endif
4933 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4934 return rc;
4935}
4936
4937
4938/**
4939 * Reports world-switch error and dumps some useful debug info.
4940 *
4941 * @param pVCpu The cross context virtual CPU structure.
4942 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4943 * @param pCtx Pointer to the guest-CPU context.
4944 * @param pVmxTransient Pointer to the VMX transient structure (only
4945 * exitReason updated).
4946 */
4947static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4948{
4949 Assert(pVCpu);
4950 Assert(pCtx);
4951 Assert(pVmxTransient);
4952 HMVMX_ASSERT_PREEMPT_SAFE();
4953
4954 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4955 switch (rcVMRun)
4956 {
4957 case VERR_VMX_INVALID_VMXON_PTR:
4958 AssertFailed();
4959 break;
4960 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4961 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4962 {
4963 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4964 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4965 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4966 AssertRC(rc);
4967
4968 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4969 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4970 Cannot do it here as we may have been long preempted. */
4971
4972#ifdef VBOX_STRICT
4973 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4974 pVmxTransient->uExitReason));
4975 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4976 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4977 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4978 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4979 else
4980 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4981 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4982 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4983
4984 /* VMX control bits. */
4985 uint32_t u32Val;
4986 uint64_t u64Val;
4987 RTHCUINTREG uHCReg;
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4992 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4993 {
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4996 }
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5009 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5010 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5012 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5013 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5014 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5016 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5018 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5020 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5022 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5024 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5025 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5026 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5027 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5028 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5029 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5030 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5031 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5032 {
5033 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5034 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5035 }
5036
5037 /* Guest bits. */
5038 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5039 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5040 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5041 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5042 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5043 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5044 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5045 {
5046 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5048 }
5049
5050 /* Host bits. */
5051 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5052 Log4(("Host CR0 %#RHr\n", uHCReg));
5053 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5054 Log4(("Host CR3 %#RHr\n", uHCReg));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5056 Log4(("Host CR4 %#RHr\n", uHCReg));
5057
5058 RTGDTR HostGdtr;
5059 PCX86DESCHC pDesc;
5060 ASMGetGDTR(&HostGdtr);
5061 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5062 Log4(("Host CS %#08x\n", u32Val));
5063 if (u32Val < HostGdtr.cbGdt)
5064 {
5065 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5066 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5067 }
5068
5069 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5070 Log4(("Host DS %#08x\n", u32Val));
5071 if (u32Val < HostGdtr.cbGdt)
5072 {
5073 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5074 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5075 }
5076
5077 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5078 Log4(("Host ES %#08x\n", u32Val));
5079 if (u32Val < HostGdtr.cbGdt)
5080 {
5081 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5082 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5083 }
5084
5085 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5086 Log4(("Host FS %#08x\n", u32Val));
5087 if (u32Val < HostGdtr.cbGdt)
5088 {
5089 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5090 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5091 }
5092
5093 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5094 Log4(("Host GS %#08x\n", u32Val));
5095 if (u32Val < HostGdtr.cbGdt)
5096 {
5097 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5098 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5099 }
5100
5101 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5102 Log4(("Host SS %#08x\n", u32Val));
5103 if (u32Val < HostGdtr.cbGdt)
5104 {
5105 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5106 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5107 }
5108
5109 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5110 Log4(("Host TR %#08x\n", u32Val));
5111 if (u32Val < HostGdtr.cbGdt)
5112 {
5113 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5114 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5115 }
5116
5117 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5118 Log4(("Host TR Base %#RHv\n", uHCReg));
5119 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5120 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5121 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5122 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5123 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5124 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5125 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5126 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5127 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5128 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5129 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5130 Log4(("Host RSP %#RHv\n", uHCReg));
5131 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5132 Log4(("Host RIP %#RHv\n", uHCReg));
5133# if HC_ARCH_BITS == 64
5134 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5135 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5136 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5137 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5138 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5139 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5140# endif
5141#endif /* VBOX_STRICT */
5142 break;
5143 }
5144
5145 default:
5146 /* Impossible */
5147 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5148 break;
5149 }
5150 NOREF(pCtx);
5151}
5152
5153
5154#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5155#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5156# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5157#endif
5158#ifdef VBOX_STRICT
5159static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5160{
5161 switch (idxField)
5162 {
5163 case VMX_VMCS_GUEST_RIP:
5164 case VMX_VMCS_GUEST_RSP:
5165 case VMX_VMCS_GUEST_SYSENTER_EIP:
5166 case VMX_VMCS_GUEST_SYSENTER_ESP:
5167 case VMX_VMCS_GUEST_GDTR_BASE:
5168 case VMX_VMCS_GUEST_IDTR_BASE:
5169 case VMX_VMCS_GUEST_CS_BASE:
5170 case VMX_VMCS_GUEST_DS_BASE:
5171 case VMX_VMCS_GUEST_ES_BASE:
5172 case VMX_VMCS_GUEST_FS_BASE:
5173 case VMX_VMCS_GUEST_GS_BASE:
5174 case VMX_VMCS_GUEST_SS_BASE:
5175 case VMX_VMCS_GUEST_LDTR_BASE:
5176 case VMX_VMCS_GUEST_TR_BASE:
5177 case VMX_VMCS_GUEST_CR3:
5178 return true;
5179 }
5180 return false;
5181}
5182
5183static bool hmR0VmxIsValidReadField(uint32_t idxField)
5184{
5185 switch (idxField)
5186 {
5187 /* Read-only fields. */
5188 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5189 return true;
5190 }
5191 /* Remaining readable fields should also be writable. */
5192 return hmR0VmxIsValidWriteField(idxField);
5193}
5194#endif /* VBOX_STRICT */
5195
5196
5197/**
5198 * Executes the specified handler in 64-bit mode.
5199 *
5200 * @returns VBox status code (no informational status codes).
5201 * @param pVCpu The cross context virtual CPU structure.
5202 * @param enmOp The operation to perform.
5203 * @param cParams Number of parameters.
5204 * @param paParam Array of 32-bit parameters.
5205 */
5206VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5207{
5208 PVM pVM = pVCpu->CTX_SUFF(pVM);
5209 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5210 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5211 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5212 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5213
5214#ifdef VBOX_STRICT
5215 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5216 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5217
5218 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5219 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5220#endif
5221
5222 /* Disable interrupts. */
5223 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5224
5225#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5226 RTCPUID idHostCpu = RTMpCpuId();
5227 CPUMR0SetLApic(pVCpu, idHostCpu);
5228#endif
5229
5230 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5231 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5232
5233 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5234 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5235 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5236
5237 /* Leave VMX Root Mode. */
5238 VMXDisable();
5239
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241
5242 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5243 CPUMSetHyperEIP(pVCpu, enmOp);
5244 for (int i = (int)cParams - 1; i >= 0; i--)
5245 CPUMPushHyper(pVCpu, paParam[i]);
5246
5247 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5248
5249 /* Call the switcher. */
5250 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5251 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5252
5253 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5254 /* Make sure the VMX instructions don't cause #UD faults. */
5255 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5256
5257 /* Re-enter VMX Root Mode */
5258 int rc2 = VMXEnable(HCPhysCpuPage);
5259 if (RT_FAILURE(rc2))
5260 {
5261 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5262 ASMSetFlags(fOldEFlags);
5263 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5264 return rc2;
5265 }
5266
5267 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5268 AssertRC(rc2);
5269 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5270 Assert(!(ASMGetFlags() & X86_EFL_IF));
5271 ASMSetFlags(fOldEFlags);
5272 return rc;
5273}
5274
5275
5276/**
5277 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5278 * supporting 64-bit guests.
5279 *
5280 * @returns VBox status code.
5281 * @param fResume Whether to VMLAUNCH or VMRESUME.
5282 * @param pCtx Pointer to the guest-CPU context.
5283 * @param pCache Pointer to the VMCS cache.
5284 * @param pVM The cross context VM structure.
5285 * @param pVCpu The cross context virtual CPU structure.
5286 */
5287DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5288{
5289 NOREF(fResume);
5290
5291 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5292 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5293
5294#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5295 pCache->uPos = 1;
5296 pCache->interPD = PGMGetInterPaeCR3(pVM);
5297 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5298#endif
5299
5300#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5301 pCache->TestIn.HCPhysCpuPage = 0;
5302 pCache->TestIn.HCPhysVmcs = 0;
5303 pCache->TestIn.pCache = 0;
5304 pCache->TestOut.HCPhysVmcs = 0;
5305 pCache->TestOut.pCache = 0;
5306 pCache->TestOut.pCtx = 0;
5307 pCache->TestOut.eflags = 0;
5308#else
5309 NOREF(pCache);
5310#endif
5311
5312 uint32_t aParam[10];
5313 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5314 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5315 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5316 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5317 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5318 aParam[5] = 0;
5319 aParam[6] = VM_RC_ADDR(pVM, pVM);
5320 aParam[7] = 0;
5321 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5322 aParam[9] = 0;
5323
5324#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5325 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5326 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5327#endif
5328 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5329
5330#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5331 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5332 Assert(pCtx->dr[4] == 10);
5333 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5334#endif
5335
5336#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5337 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5338 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5339 pVCpu->hm.s.vmx.HCPhysVmcs));
5340 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5341 pCache->TestOut.HCPhysVmcs));
5342 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5343 pCache->TestOut.pCache));
5344 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5345 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5346 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5347 pCache->TestOut.pCtx));
5348 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5349#endif
5350 NOREF(pCtx);
5351 return rc;
5352}
5353
5354
5355/**
5356 * Initialize the VMCS-Read cache.
5357 *
5358 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5359 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5360 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5361 * (those that have a 32-bit FULL & HIGH part).
5362 *
5363 * @returns VBox status code.
5364 * @param pVCpu The cross context virtual CPU structure.
5365 */
5366static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5367{
5368#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5369 do { \
5370 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5371 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5372 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5373 ++cReadFields; \
5374 } while (0)
5375
5376 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5377 uint32_t cReadFields = 0;
5378
5379 /*
5380 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5381 * and serve to indicate exceptions to the rules.
5382 */
5383
5384 /* Guest-natural selector base fields. */
5385#if 0
5386 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5389#endif
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5402#if 0
5403 /* Unused natural width guest-state fields. */
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5406#endif
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5409
5410 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5411 these 64-bit fields (using "FULL" and "HIGH" fields). */
5412#if 0
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5422#endif
5423
5424 /* Natural width guest-state fields. */
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5426#if 0
5427 /* Currently unused field. */
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5429#endif
5430
5431 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5432 {
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5434 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5435 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5436 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5437 }
5438 else
5439 {
5440 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5441 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5442 }
5443
5444#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5445 return VINF_SUCCESS;
5446}
5447
5448
5449/**
5450 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5451 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5452 * darwin, running 64-bit guests).
5453 *
5454 * @returns VBox status code.
5455 * @param pVCpu The cross context virtual CPU structure.
5456 * @param idxField The VMCS field encoding.
5457 * @param u64Val 16, 32 or 64-bit value.
5458 */
5459VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5460{
5461 int rc;
5462 switch (idxField)
5463 {
5464 /*
5465 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5466 */
5467 /* 64-bit Control fields. */
5468 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5469 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5470 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5471 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5472 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5473 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5474 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5475 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5476 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5477 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5478 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5479 case VMX_VMCS64_CTRL_EPTP_FULL:
5480 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5481 /* 64-bit Guest-state fields. */
5482 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5483 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5484 case VMX_VMCS64_GUEST_PAT_FULL:
5485 case VMX_VMCS64_GUEST_EFER_FULL:
5486 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5487 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5488 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5489 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5490 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5491 /* 64-bit Host-state fields. */
5492 case VMX_VMCS64_HOST_PAT_FULL:
5493 case VMX_VMCS64_HOST_EFER_FULL:
5494 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5495 {
5496 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5497 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5498 break;
5499 }
5500
5501 /*
5502 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5503 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5504 */
5505 /* Natural-width Guest-state fields. */
5506 case VMX_VMCS_GUEST_CR3:
5507 case VMX_VMCS_GUEST_ES_BASE:
5508 case VMX_VMCS_GUEST_CS_BASE:
5509 case VMX_VMCS_GUEST_SS_BASE:
5510 case VMX_VMCS_GUEST_DS_BASE:
5511 case VMX_VMCS_GUEST_FS_BASE:
5512 case VMX_VMCS_GUEST_GS_BASE:
5513 case VMX_VMCS_GUEST_LDTR_BASE:
5514 case VMX_VMCS_GUEST_TR_BASE:
5515 case VMX_VMCS_GUEST_GDTR_BASE:
5516 case VMX_VMCS_GUEST_IDTR_BASE:
5517 case VMX_VMCS_GUEST_RSP:
5518 case VMX_VMCS_GUEST_RIP:
5519 case VMX_VMCS_GUEST_SYSENTER_ESP:
5520 case VMX_VMCS_GUEST_SYSENTER_EIP:
5521 {
5522 if (!(RT_HI_U32(u64Val)))
5523 {
5524 /* If this field is 64-bit, VT-x will zero out the top bits. */
5525 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5526 }
5527 else
5528 {
5529 /* Assert that only the 32->64 switcher case should ever come here. */
5530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5531 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5532 }
5533 break;
5534 }
5535
5536 default:
5537 {
5538 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5539 rc = VERR_INVALID_PARAMETER;
5540 break;
5541 }
5542 }
5543 AssertRCReturn(rc, rc);
5544 return rc;
5545}
5546
5547
5548/**
5549 * Queue up a VMWRITE by using the VMCS write cache.
5550 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5551 *
5552 * @param pVCpu The cross context virtual CPU structure.
5553 * @param idxField The VMCS field encoding.
5554 * @param u64Val 16, 32 or 64-bit value.
5555 */
5556VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5557{
5558 AssertPtr(pVCpu);
5559 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5560
5561 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5562 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5563
5564 /* Make sure there are no duplicates. */
5565 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5566 {
5567 if (pCache->Write.aField[i] == idxField)
5568 {
5569 pCache->Write.aFieldVal[i] = u64Val;
5570 return VINF_SUCCESS;
5571 }
5572 }
5573
5574 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5575 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5576 pCache->Write.cValidEntries++;
5577 return VINF_SUCCESS;
5578}
5579#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5580
5581
5582/**
5583 * Sets up the usage of TSC-offsetting and updates the VMCS.
5584 *
5585 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5586 * VMX preemption timer.
5587 *
5588 * @returns VBox status code.
5589 * @param pVCpu The cross context virtual CPU structure.
5590 *
5591 * @remarks No-long-jump zone!!!
5592 */
5593static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5594{
5595 bool fOffsettedTsc;
5596 bool fParavirtTsc;
5597 PVM pVM = pVCpu->CTX_SUFF(pVM);
5598 uint64_t uTscOffset;
5599 if (pVM->hm.s.vmx.fUsePreemptTimer)
5600 {
5601 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5602
5603 /* Make sure the returned values have sane upper and lower boundaries. */
5604 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5605 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5606 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5607 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5608
5609 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5610 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5611 AssertRC(rc);
5612 }
5613 else
5614 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5615
5616 /** @todo later optimize this to be done elsewhere and not before every
5617 * VM-entry. */
5618 if (fParavirtTsc)
5619 {
5620 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5621 information before every VM-entry, hence disable it for performance sake. */
5622#if 0
5623 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5624 AssertRC(rc);
5625#endif
5626 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5627 }
5628
5629 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5630 if ( fOffsettedTsc
5631 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5632 {
5633 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5634 {
5635 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5636 AssertRC(rc);
5637 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5638 }
5639
5640 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5641 {
5642 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5643 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5644 AssertRC(rc);
5645 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5646 }
5647 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5648 }
5649 else
5650 {
5651 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5652 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5653 {
5654 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5655 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5656 AssertRC(rc);
5657 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5658 }
5659 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5660 }
5661}
5662
5663
5664/**
5665 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5666 * VM-exit interruption info type.
5667 *
5668 * @returns The IEM exception flags.
5669 * @param uVector The event vector.
5670 * @param uVmxVectorType The VMX event type.
5671 *
5672 * @remarks This function currently only constructs flags required for
5673 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5674 * and CR2 aspects of an exception are not included).
5675 */
5676static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5677{
5678 uint32_t fIemXcptFlags;
5679 switch (uVmxVectorType)
5680 {
5681 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5682 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5683 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5684 break;
5685
5686 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5687 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5688 break;
5689
5690 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5691 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5692 break;
5693
5694 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5695 {
5696 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5697 if (uVector == X86_XCPT_BP)
5698 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5699 else if (uVector == X86_XCPT_OF)
5700 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5701 else
5702 {
5703 fIemXcptFlags = 0;
5704 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5705 }
5706 break;
5707 }
5708
5709 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5710 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5711 break;
5712
5713 default:
5714 fIemXcptFlags = 0;
5715 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5716 break;
5717 }
5718 return fIemXcptFlags;
5719}
5720
5721
5722/**
5723 * Sets an event as a pending event to be injected into the guest.
5724 *
5725 * @param pVCpu The cross context virtual CPU structure.
5726 * @param u32IntInfo The VM-entry interruption-information field.
5727 * @param cbInstr The VM-entry instruction length in bytes (for software
5728 * interrupts, exceptions and privileged software
5729 * exceptions).
5730 * @param u32ErrCode The VM-entry exception error code.
5731 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5732 * page-fault.
5733 *
5734 * @remarks Statistics counter assumes this is a guest event being injected or
5735 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5736 * always incremented.
5737 */
5738DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5739 RTGCUINTPTR GCPtrFaultAddress)
5740{
5741 Assert(!pVCpu->hm.s.Event.fPending);
5742 pVCpu->hm.s.Event.fPending = true;
5743 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5744 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5745 pVCpu->hm.s.Event.cbInstr = cbInstr;
5746 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5747}
5748
5749
5750/**
5751 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5752 *
5753 * @param pVCpu The cross context virtual CPU structure.
5754 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5755 * out-of-sync. Make sure to update the required fields
5756 * before using them.
5757 */
5758DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5759{
5760 NOREF(pMixedCtx);
5761 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5762 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5763 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5764 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5765}
5766
5767
5768/**
5769 * Handle a condition that occurred while delivering an event through the guest
5770 * IDT.
5771 *
5772 * @returns Strict VBox status code (i.e. informational status codes too).
5773 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5774 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5775 * to continue execution of the guest which will delivery the \#DF.
5776 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5777 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5778 *
5779 * @param pVCpu The cross context virtual CPU structure.
5780 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5781 * out-of-sync. Make sure to update the required fields
5782 * before using them.
5783 * @param pVmxTransient Pointer to the VMX transient structure.
5784 *
5785 * @remarks No-long-jump zone!!!
5786 */
5787static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5788{
5789 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5790
5791 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5792 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5793 AssertRCReturn(rc2, rc2);
5794
5795 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5796 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5797 {
5798 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5799 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5800
5801 /*
5802 * If the event was a software interrupt (generated with INT n) or a software exception
5803 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5804 * can handle the VM-exit and continue guest execution which will re-execute the
5805 * instruction rather than re-injecting the exception, as that can cause premature
5806 * trips to ring-3 before injection and involve TRPM which currently has no way of
5807 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5808 * the problem).
5809 */
5810 IEMXCPTRAISE enmRaise;
5811 IEMXCPTRAISEINFO fRaiseInfo;
5812 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5813 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5814 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5815 {
5816 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5817 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5818 }
5819 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5820 {
5821 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5822 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5823 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5824 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5825 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5826 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5827 uExitVectorType), VERR_VMX_IPE_5);
5828
5829 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5830
5831 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5832 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5833 {
5834 pVmxTransient->fVectoringPF = true;
5835 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5836 }
5837 }
5838 else
5839 {
5840 /*
5841 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5842 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5843 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5844 */
5845 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5846 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5847 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5848 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5849 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5850 }
5851
5852 /*
5853 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5854 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5855 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5856 * subsequent VM-entry would fail.
5857 *
5858 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5859 */
5860 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5861 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5862 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5863 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5864 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5865 {
5866 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5867 }
5868
5869 switch (enmRaise)
5870 {
5871 case IEMXCPTRAISE_CURRENT_XCPT:
5872 {
5873 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5874 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5875 Assert(rcStrict == VINF_SUCCESS);
5876 break;
5877 }
5878
5879 case IEMXCPTRAISE_PREV_EVENT:
5880 {
5881 uint32_t u32ErrCode;
5882 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5883 {
5884 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5885 AssertRCReturn(rc2, rc2);
5886 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5887 }
5888 else
5889 u32ErrCode = 0;
5890
5891 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5892 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5893 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5894 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5895
5896 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5897 pVCpu->hm.s.Event.u32ErrCode));
5898 Assert(rcStrict == VINF_SUCCESS);
5899 break;
5900 }
5901
5902 case IEMXCPTRAISE_REEXEC_INSTR:
5903 Assert(rcStrict == VINF_SUCCESS);
5904 break;
5905
5906 case IEMXCPTRAISE_DOUBLE_FAULT:
5907 {
5908 /*
5909 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5910 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5911 */
5912 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5913 {
5914 pVmxTransient->fVectoringDoublePF = true;
5915 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5916 pMixedCtx->cr2));
5917 rcStrict = VINF_SUCCESS;
5918 }
5919 else
5920 {
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5923 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5924 uIdtVector, uExitVector));
5925 rcStrict = VINF_HM_DOUBLE_FAULT;
5926 }
5927 break;
5928 }
5929
5930 case IEMXCPTRAISE_TRIPLE_FAULT:
5931 {
5932 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5933 rcStrict = VINF_EM_RESET;
5934 break;
5935 }
5936
5937 case IEMXCPTRAISE_CPU_HANG:
5938 {
5939 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5940 rcStrict = VERR_EM_GUEST_CPU_HANG;
5941 break;
5942 }
5943
5944 default:
5945 {
5946 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5947 rcStrict = VERR_VMX_IPE_2;
5948 break;
5949 }
5950 }
5951 }
5952 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5953 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5954 && uExitVector != X86_XCPT_DF
5955 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5956 {
5957 /*
5958 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5959 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5960 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5961 */
5962 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5963 {
5964 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5965 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5966 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5967 }
5968 }
5969
5970 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5971 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5972 return rcStrict;
5973}
5974
5975
5976/**
5977 * Imports a guest segment register from the current VMCS into
5978 * the guest-CPU context.
5979 *
5980 * @returns VBox status code.
5981 * @param pVCpu The cross context virtual CPU structure.
5982 * @param idxSel Index of the selector in the VMCS.
5983 * @param idxLimit Index of the segment limit in the VMCS.
5984 * @param idxBase Index of the segment base in the VMCS.
5985 * @param idxAccess Index of the access rights of the segment in the VMCS.
5986 * @param pSelReg Pointer to the segment selector.
5987 *
5988 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
5989 * do not log!
5990 *
5991 * @remarks Never call this function directly!!! Use the
5992 * HMVMX_IMPORT_SREG() macro as that takes care
5993 * of whether to read from the VMCS cache or not.
5994 */
5995static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5996 PCPUMSELREG pSelReg)
5997{
5998 NOREF(pVCpu);
5999
6000 uint32_t u32Sel;
6001 uint32_t u32Limit;
6002 uint32_t u32Attr;
6003 uint64_t u64Base;
6004 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6005 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6006 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6007 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6008 AssertRCReturn(rc, rc);
6009
6010 pSelReg->Sel = (uint16_t)u32Sel;
6011 pSelReg->ValidSel = (uint16_t)u32Sel;
6012 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6013 pSelReg->u32Limit = u32Limit;
6014 pSelReg->u64Base = u64Base;
6015 pSelReg->Attr.u = u32Attr;
6016
6017 /*
6018 * If VT-x marks the segment as unusable, most other bits remain undefined:
6019 * - For CS the L, D and G bits have meaning.
6020 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6021 * - For the remaining data segments no bits are defined.
6022 *
6023 * The present bit and the unusable bit has been observed to be set at the
6024 * same time (the selector was supposed to be invalid as we started executing
6025 * a V8086 interrupt in ring-0).
6026 *
6027 * What should be important for the rest of the VBox code, is that the P bit is
6028 * cleared. Some of the other VBox code recognizes the unusable bit, but
6029 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6030 * safe side here, we'll strip off P and other bits we don't care about. If
6031 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6032 *
6033 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6034 */
6035 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6036 {
6037 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6038
6039 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6040 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6041 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6042#ifdef VBOX_STRICT
6043 VMMRZCallRing3Disable(pVCpu);
6044 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6045# ifdef DEBUG_bird
6046 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6047 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6048 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6049# endif
6050 VMMRZCallRing3Enable(pVCpu);
6051#endif
6052 }
6053 return VINF_SUCCESS;
6054}
6055
6056
6057/**
6058 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6059 *
6060 * @returns VBox status code.
6061 * @param pVCpu The cross context virtual CPU structure.
6062 *
6063 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6064 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6065 * instead!!!
6066 */
6067DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6068{
6069 uint64_t u64Val;
6070 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6071 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6072 {
6073 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6074 if (RT_SUCCESS(rc))
6075 {
6076 pCtx->rip = u64Val;
6077 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6078 }
6079 return rc;
6080 }
6081 return VINF_SUCCESS;
6082}
6083
6084
6085/**
6086 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6087 *
6088 * @returns VBox status code.
6089 * @param pVCpu The cross context virtual CPU structure.
6090 *
6091 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6092 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6093 * instead!!!
6094 */
6095DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6096{
6097 uint32_t u32Val;
6098 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6099 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6100 {
6101 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6102 if (RT_SUCCESS(rc))
6103 {
6104 pCtx->eflags.u32 = u32Val;
6105
6106 /* Restore eflags for real-on-v86-mode hack. */
6107 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6108 {
6109 pCtx->eflags.Bits.u1VM = 0;
6110 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6111 }
6112 }
6113 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6114 return rc;
6115 }
6116 return VINF_SUCCESS;
6117}
6118
6119
6120/**
6121 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6122 * context.
6123 *
6124 * @returns VBox status code.
6125 * @param pVCpu The cross context virtual CPU structure.
6126 *
6127 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6128 * do not log!
6129 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6130 * instead!!!
6131 */
6132DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6133{
6134 uint32_t u32Val;
6135 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6136 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6137 if (RT_SUCCESS(rc))
6138 {
6139 /*
6140 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6141 * might need them in hmR0VmxEvaluatePendingEvent().
6142 */
6143 if (!u32Val)
6144 {
6145 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6146 {
6147 rc = hmR0VmxImportGuestRip(pVCpu);
6148 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6149 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6150 }
6151
6152 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6153 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6154 }
6155 else
6156 {
6157 rc = hmR0VmxImportGuestRip(pVCpu);
6158 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6159
6160 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6161 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6162 {
6163 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6164 }
6165 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6166 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6167
6168 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6169 {
6170 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6171 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6172 }
6173 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6174 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6175 }
6176 }
6177 return rc;
6178}
6179
6180
6181/**
6182 * Worker for VMXR0ImportStateOnDemand.
6183 *
6184 * @returns VBox status code.
6185 * @param pVCpu The cross context virtual CPU structure.
6186 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6187 */
6188static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6189{
6190#define VMXLOCAL_BREAK_RC(a_rc) \
6191 if (RT_FAILURE(a_rc)) \
6192 break
6193
6194 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6195
6196 int rc = VINF_SUCCESS;
6197 PVM pVM = pVCpu->CTX_SUFF(pVM);
6198 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6199 uint64_t u64Val;
6200 uint32_t u32Val;
6201
6202 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6203
6204 /*
6205 * We disable interrupts to make the updating of the state and in particular
6206 * the fExtrn modification atomic wrt to preemption hooks.
6207 */
6208 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6209
6210 fWhat &= pCtx->fExtrn;
6211 if (fWhat)
6212 {
6213 do
6214 {
6215 if (fWhat & CPUMCTX_EXTRN_RIP)
6216 {
6217 rc = hmR0VmxImportGuestRip(pVCpu);
6218 VMXLOCAL_BREAK_RC(rc);
6219 }
6220
6221 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6222 {
6223 rc = hmR0VmxImportGuestRFlags(pVCpu);
6224 VMXLOCAL_BREAK_RC(rc);
6225 }
6226
6227 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6228 {
6229 rc = hmR0VmxImportGuestIntrState(pVCpu);
6230 VMXLOCAL_BREAK_RC(rc);
6231 }
6232
6233 if (fWhat & CPUMCTX_EXTRN_RSP)
6234 {
6235 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6236 VMXLOCAL_BREAK_RC(rc);
6237 pCtx->rsp = u64Val;
6238 }
6239
6240 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6241 {
6242 if (fWhat & CPUMCTX_EXTRN_CS)
6243 {
6244 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6245 VMXLOCAL_BREAK_RC(rc);
6246 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6247 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6248 }
6249 if (fWhat & CPUMCTX_EXTRN_SS)
6250 {
6251 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6252 VMXLOCAL_BREAK_RC(rc);
6253 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6254 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6255 }
6256 if (fWhat & CPUMCTX_EXTRN_DS)
6257 {
6258 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6259 VMXLOCAL_BREAK_RC(rc);
6260 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6261 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6262 }
6263 if (fWhat & CPUMCTX_EXTRN_ES)
6264 {
6265 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6266 VMXLOCAL_BREAK_RC(rc);
6267 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6268 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6269 }
6270 if (fWhat & CPUMCTX_EXTRN_FS)
6271 {
6272 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6273 VMXLOCAL_BREAK_RC(rc);
6274 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6275 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6276 }
6277 if (fWhat & CPUMCTX_EXTRN_GS)
6278 {
6279 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6280 VMXLOCAL_BREAK_RC(rc);
6281 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6282 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6283 }
6284 }
6285
6286 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6287 {
6288 if (fWhat & CPUMCTX_EXTRN_LDTR)
6289 {
6290 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6291 VMXLOCAL_BREAK_RC(rc);
6292 }
6293
6294 if (fWhat & CPUMCTX_EXTRN_GDTR)
6295 {
6296 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6297 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6298 VMXLOCAL_BREAK_RC(rc);
6299 pCtx->gdtr.pGdt = u64Val;
6300 pCtx->gdtr.cbGdt = u32Val;
6301 }
6302
6303 /* Guest IDTR. */
6304 if (fWhat & CPUMCTX_EXTRN_IDTR)
6305 {
6306 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6307 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6308 VMXLOCAL_BREAK_RC(rc);
6309 pCtx->idtr.pIdt = u64Val;
6310 pCtx->idtr.cbIdt = u32Val;
6311 }
6312
6313 /* Guest TR. */
6314 if (fWhat & CPUMCTX_EXTRN_TR)
6315 {
6316 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6317 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6318 {
6319 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6320 VMXLOCAL_BREAK_RC(rc);
6321 }
6322 }
6323 }
6324
6325 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6326 {
6327 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6328 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6329 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6330 pCtx->SysEnter.cs = u32Val;
6331 VMXLOCAL_BREAK_RC(rc);
6332 }
6333
6334#if HC_ARCH_BITS == 64
6335 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6336 {
6337 if ( pVM->hm.s.fAllow64BitGuests
6338 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6339 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6340 }
6341
6342 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6343 {
6344 if ( pVM->hm.s.fAllow64BitGuests
6345 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6346 {
6347 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6348 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6349 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6350 }
6351 }
6352#endif
6353
6354 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6355#if HC_ARCH_BITS == 32
6356 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6357#endif
6358 )
6359 {
6360 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6361 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6362 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6363 {
6364 switch (pMsr->u32Msr)
6365 {
6366#if HC_ARCH_BITS == 32
6367 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6368 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6369 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6370 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6371#endif
6372 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6373 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6374 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6375 default:
6376 {
6377 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6378 ASMSetFlags(fEFlags);
6379 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6380 cMsrs));
6381 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6382 }
6383 }
6384 }
6385 }
6386
6387 if (fWhat & CPUMCTX_EXTRN_DR7)
6388 {
6389 if (!pVCpu->hm.s.fUsingHyperDR7)
6390 {
6391 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6392 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6393 VMXLOCAL_BREAK_RC(rc);
6394 pCtx->dr[7] = u32Val;
6395 }
6396 }
6397
6398 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6399 {
6400 uint32_t u32Shadow;
6401 if (fWhat & CPUMCTX_EXTRN_CR0)
6402 {
6403 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6404 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6405 VMXLOCAL_BREAK_RC(rc);
6406 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6407 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6408 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6409 CPUMSetGuestCR0(pVCpu, u32Val);
6410 VMMRZCallRing3Enable(pVCpu);
6411 }
6412
6413 if (fWhat & CPUMCTX_EXTRN_CR4)
6414 {
6415 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6416 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6417 VMXLOCAL_BREAK_RC(rc);
6418 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6419 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6420 CPUMSetGuestCR4(pVCpu, u32Val);
6421 }
6422
6423 if (fWhat & CPUMCTX_EXTRN_CR3)
6424 {
6425 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6426 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6427 || ( pVM->hm.s.fNestedPaging
6428 && CPUMIsGuestPagingEnabledEx(pCtx)))
6429 {
6430 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6431 if (pCtx->cr3 != u64Val)
6432 {
6433 CPUMSetGuestCR3(pVCpu, u64Val);
6434 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6435 }
6436
6437 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6438 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6439 if (CPUMIsGuestInPAEModeEx(pCtx))
6440 {
6441 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6442 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6443 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6444 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6445 VMXLOCAL_BREAK_RC(rc);
6446 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6447 }
6448 }
6449 }
6450 }
6451 } while (0);
6452
6453 if (RT_SUCCESS(rc))
6454 {
6455 /* Update fExtrn. */
6456 pCtx->fExtrn &= ~fWhat;
6457
6458 /* If everything has been imported, clear the HM keeper bit. */
6459 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6460 {
6461 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6462 Assert(!pCtx->fExtrn);
6463 }
6464 }
6465 }
6466 else
6467 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6468
6469 ASMSetFlags(fEFlags);
6470
6471 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6472
6473 /*
6474 * Honor any pending CR3 updates.
6475 *
6476 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6477 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6478 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6479 *
6480 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6481 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6482 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6483 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6484 *
6485 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6486 */
6487 if (VMMRZCallRing3IsEnabled(pVCpu))
6488 {
6489 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6490 {
6491 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6492 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6493 }
6494
6495 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6496 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6497
6498 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6499 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6500 }
6501
6502 return VINF_SUCCESS;
6503#undef VMXLOCAL_BREAK_RC
6504}
6505
6506
6507/**
6508 * Saves the guest state from the VMCS into the guest-CPU context.
6509 *
6510 * @returns VBox status code.
6511 * @param pVCpu The cross context virtual CPU structure.
6512 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6513 */
6514VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6515{
6516 return hmR0VmxImportGuestState(pVCpu, fWhat);
6517}
6518
6519
6520/**
6521 * Check per-VM and per-VCPU force flag actions that require us to go back to
6522 * ring-3 for one reason or another.
6523 *
6524 * @returns Strict VBox status code (i.e. informational status codes too)
6525 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6526 * ring-3.
6527 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6528 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6529 * interrupts)
6530 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6531 * all EMTs to be in ring-3.
6532 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6533 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6534 * to the EM loop.
6535 *
6536 * @param pVCpu The cross context virtual CPU structure.
6537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6538 * out-of-sync. Make sure to update the required fields
6539 * before using them.
6540 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6541 */
6542static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6543{
6544 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6545
6546 /*
6547 * Anything pending? Should be more likely than not if we're doing a good job.
6548 */
6549 PVM pVM = pVCpu->CTX_SUFF(pVM);
6550 if ( !fStepping
6551 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6552 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6553 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6554 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6555 return VINF_SUCCESS;
6556
6557 /* Pending PGM C3 sync. */
6558 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6559 {
6560 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6561 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6562 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6563 if (rcStrict2 != VINF_SUCCESS)
6564 {
6565 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6566 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6567 return rcStrict2;
6568 }
6569 }
6570
6571 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6572 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6573 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6574 {
6575 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6576 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6577 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6578 return rc2;
6579 }
6580
6581 /* Pending VM request packets, such as hardware interrupts. */
6582 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6583 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6584 {
6585 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6586 return VINF_EM_PENDING_REQUEST;
6587 }
6588
6589 /* Pending PGM pool flushes. */
6590 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6591 {
6592 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6593 return VINF_PGM_POOL_FLUSH_PENDING;
6594 }
6595
6596 /* Pending DMA requests. */
6597 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6598 {
6599 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6600 return VINF_EM_RAW_TO_R3;
6601 }
6602
6603 return VINF_SUCCESS;
6604}
6605
6606
6607/**
6608 * Converts any TRPM trap into a pending HM event. This is typically used when
6609 * entering from ring-3 (not longjmp returns).
6610 *
6611 * @param pVCpu The cross context virtual CPU structure.
6612 */
6613static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6614{
6615 Assert(TRPMHasTrap(pVCpu));
6616 Assert(!pVCpu->hm.s.Event.fPending);
6617
6618 uint8_t uVector;
6619 TRPMEVENT enmTrpmEvent;
6620 RTGCUINT uErrCode;
6621 RTGCUINTPTR GCPtrFaultAddress;
6622 uint8_t cbInstr;
6623
6624 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6625 AssertRC(rc);
6626
6627 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6628 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6629 if (enmTrpmEvent == TRPM_TRAP)
6630 {
6631 switch (uVector)
6632 {
6633 case X86_XCPT_NMI:
6634 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6635 break;
6636
6637 case X86_XCPT_BP:
6638 case X86_XCPT_OF:
6639 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6640 break;
6641
6642 case X86_XCPT_PF:
6643 case X86_XCPT_DF:
6644 case X86_XCPT_TS:
6645 case X86_XCPT_NP:
6646 case X86_XCPT_SS:
6647 case X86_XCPT_GP:
6648 case X86_XCPT_AC:
6649 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6650 RT_FALL_THRU();
6651 default:
6652 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6653 break;
6654 }
6655 }
6656 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6657 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6658 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6659 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6660 else
6661 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6662
6663 rc = TRPMResetTrap(pVCpu);
6664 AssertRC(rc);
6665 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6666 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6667
6668 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6669}
6670
6671
6672/**
6673 * Converts the pending HM event into a TRPM trap.
6674 *
6675 * @param pVCpu The cross context virtual CPU structure.
6676 */
6677static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6678{
6679 Assert(pVCpu->hm.s.Event.fPending);
6680
6681 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6682 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6683 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6684 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6685
6686 /* If a trap was already pending, we did something wrong! */
6687 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6688
6689 TRPMEVENT enmTrapType;
6690 switch (uVectorType)
6691 {
6692 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6693 enmTrapType = TRPM_HARDWARE_INT;
6694 break;
6695
6696 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6697 enmTrapType = TRPM_SOFTWARE_INT;
6698 break;
6699
6700 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6701 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6702 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6703 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6704 enmTrapType = TRPM_TRAP;
6705 break;
6706
6707 default:
6708 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6709 enmTrapType = TRPM_32BIT_HACK;
6710 break;
6711 }
6712
6713 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6714
6715 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6716 AssertRC(rc);
6717
6718 if (fErrorCodeValid)
6719 TRPMSetErrorCode(pVCpu, uErrorCode);
6720
6721 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6722 && uVector == X86_XCPT_PF)
6723 {
6724 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6725 }
6726 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6727 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6728 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6729 {
6730 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6731 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6732 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6733 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6734 }
6735
6736 /* Clear any pending events from the VMCS. */
6737 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6738 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6739
6740 /* We're now done converting the pending event. */
6741 pVCpu->hm.s.Event.fPending = false;
6742}
6743
6744
6745/**
6746 * Does the necessary state syncing before returning to ring-3 for any reason
6747 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6748 *
6749 * @returns VBox status code.
6750 * @param pVCpu The cross context virtual CPU structure.
6751 * @param fImportState Whether to import the guest state from the VMCS back
6752 * to the guest-CPU context.
6753 *
6754 * @remarks No-long-jmp zone!!!
6755 */
6756static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6757{
6758 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6759 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6760
6761 RTCPUID idCpu = RTMpCpuId();
6762 Log4Func(("HostCpuId=%u\n", idCpu));
6763
6764 /*
6765 * !!! IMPORTANT !!!
6766 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6767 */
6768
6769 /* Save the guest state if necessary. */
6770 if (fImportState)
6771 {
6772 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6773 AssertRCReturn(rc, rc);
6774 }
6775
6776 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6777 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6778 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6779
6780 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6781#ifdef VBOX_STRICT
6782 if (CPUMIsHyperDebugStateActive(pVCpu))
6783 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6784#endif
6785 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6786 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6787 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6788
6789#if HC_ARCH_BITS == 64
6790 /* Restore host-state bits that VT-x only restores partially. */
6791 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6792 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6793 {
6794 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6795 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6796 }
6797 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6798#endif
6799
6800 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6801 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6802 {
6803 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6804 if (!fImportState)
6805 {
6806 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6807 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6808 AssertRCReturn(rc, rc);
6809 }
6810 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6811 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6812 }
6813 else
6814 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6815
6816 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6817 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6818
6819 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6820 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6821 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6822 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6823 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6824 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6825 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6826 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6827 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6828
6829 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6830
6831 /** @todo This partially defeats the purpose of having preemption hooks.
6832 * The problem is, deregistering the hooks should be moved to a place that
6833 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6834 * context.
6835 */
6836 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6837 {
6838 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6839 AssertRCReturn(rc, rc);
6840
6841 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6842 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6843 }
6844 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6845 NOREF(idCpu);
6846
6847 return VINF_SUCCESS;
6848}
6849
6850
6851/**
6852 * Leaves the VT-x session.
6853 *
6854 * @returns VBox status code.
6855 * @param pVCpu The cross context virtual CPU structure.
6856 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6857 * out-of-sync. Make sure to update the required fields
6858 * before using them.
6859 *
6860 * @remarks No-long-jmp zone!!!
6861 */
6862static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6863{
6864 HM_DISABLE_PREEMPT();
6865 HMVMX_ASSERT_CPU_SAFE();
6866 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6867 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6868
6869 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6870 and done this from the VMXR0ThreadCtxCallback(). */
6871 if (!pVCpu->hm.s.fLeaveDone)
6872 {
6873 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6874 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6875 pVCpu->hm.s.fLeaveDone = true;
6876 }
6877 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6878
6879 /*
6880 * !!! IMPORTANT !!!
6881 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6882 */
6883
6884 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6885 /** @todo Deregistering here means we need to VMCLEAR always
6886 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6887 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6888 VMMR0ThreadCtxHookDisable(pVCpu);
6889
6890 /* Leave HM context. This takes care of local init (term). */
6891 int rc = HMR0LeaveCpu(pVCpu);
6892
6893 HM_RESTORE_PREEMPT();
6894 return rc;
6895}
6896
6897
6898/**
6899 * Does the necessary state syncing before doing a longjmp to ring-3.
6900 *
6901 * @returns VBox status code.
6902 * @param pVCpu The cross context virtual CPU structure.
6903 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6904 * out-of-sync. Make sure to update the required fields
6905 * before using them.
6906 *
6907 * @remarks No-long-jmp zone!!!
6908 */
6909DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6910{
6911 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6912}
6913
6914
6915/**
6916 * Take necessary actions before going back to ring-3.
6917 *
6918 * An action requires us to go back to ring-3. This function does the necessary
6919 * steps before we can safely return to ring-3. This is not the same as longjmps
6920 * to ring-3, this is voluntary and prepares the guest so it may continue
6921 * executing outside HM (recompiler/IEM).
6922 *
6923 * @returns VBox status code.
6924 * @param pVCpu The cross context virtual CPU structure.
6925 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6926 * out-of-sync. Make sure to update the required fields
6927 * before using them.
6928 * @param rcExit The reason for exiting to ring-3. Can be
6929 * VINF_VMM_UNKNOWN_RING3_CALL.
6930 */
6931static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6932{
6933 Assert(pVCpu);
6934 Assert(pMixedCtx);
6935 HMVMX_ASSERT_PREEMPT_SAFE();
6936
6937 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6938 {
6939 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6940 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6941 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6942 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6943 }
6944
6945 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6946 VMMRZCallRing3Disable(pVCpu);
6947 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6948
6949 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6950 if (pVCpu->hm.s.Event.fPending)
6951 {
6952 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6953 Assert(!pVCpu->hm.s.Event.fPending);
6954 }
6955
6956 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6957 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6958
6959 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6960 and if we're injecting an event we should have a TRPM trap pending. */
6961 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6962#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6963 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6964#endif
6965
6966 /* Save guest state and restore host state bits. */
6967 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6968 AssertRCReturn(rc, rc);
6969 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6970 /* Thread-context hooks are unregistered at this point!!! */
6971
6972 /* Sync recompiler state. */
6973 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6974 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6975 | CPUM_CHANGED_LDTR
6976 | CPUM_CHANGED_GDTR
6977 | CPUM_CHANGED_IDTR
6978 | CPUM_CHANGED_TR
6979 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6980 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6981 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6982 {
6983 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6984 }
6985
6986 Assert(!pVCpu->hm.s.fClearTrapFlag);
6987
6988 /* Update the exit-to-ring 3 reason. */
6989 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6990
6991 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6992 if (rcExit != VINF_EM_RAW_INTERRUPT)
6993 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6994
6995 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6996
6997 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6998 VMMRZCallRing3RemoveNotification(pVCpu);
6999 VMMRZCallRing3Enable(pVCpu);
7000
7001 return rc;
7002}
7003
7004
7005/**
7006 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7007 * longjump to ring-3 and possibly get preempted.
7008 *
7009 * @returns VBox status code.
7010 * @param pVCpu The cross context virtual CPU structure.
7011 * @param enmOperation The operation causing the ring-3 longjump.
7012 * @param pvUser Opaque pointer to the guest-CPU context. The data
7013 * may be out-of-sync. Make sure to update the required
7014 * fields before using them.
7015 */
7016static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7017{
7018 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7019 {
7020 /*
7021 * !!! IMPORTANT !!!
7022 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7023 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7024 */
7025 VMMRZCallRing3RemoveNotification(pVCpu);
7026 VMMRZCallRing3Disable(pVCpu);
7027 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7028 RTThreadPreemptDisable(&PreemptState);
7029
7030 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7031 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7032
7033#if HC_ARCH_BITS == 64
7034 /* Restore host-state bits that VT-x only restores partially. */
7035 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7036 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7037 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7038 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7039#endif
7040
7041 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7042 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7043 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7044
7045 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7046 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7047 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7048 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7049 {
7050 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7051 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7052 }
7053
7054 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7055 VMMR0ThreadCtxHookDisable(pVCpu);
7056 HMR0LeaveCpu(pVCpu);
7057 RTThreadPreemptRestore(&PreemptState);
7058 return VINF_SUCCESS;
7059 }
7060
7061 Assert(pVCpu);
7062 Assert(pvUser);
7063 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7064 HMVMX_ASSERT_PREEMPT_SAFE();
7065
7066 VMMRZCallRing3Disable(pVCpu);
7067 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7068
7069 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7070
7071 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7072 AssertRCReturn(rc, rc);
7073
7074 VMMRZCallRing3Enable(pVCpu);
7075 return VINF_SUCCESS;
7076}
7077
7078
7079/**
7080 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7081 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7082 *
7083 * @param pVCpu The cross context virtual CPU structure.
7084 */
7085DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7086{
7087 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7088 {
7089 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7090 {
7091 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7092 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7093 AssertRC(rc);
7094 Log4Func(("Setup interrupt-window exiting\n"));
7095 }
7096 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7097}
7098
7099
7100/**
7101 * Clears the interrupt-window exiting control in the VMCS.
7102 *
7103 * @param pVCpu The cross context virtual CPU structure.
7104 */
7105DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7106{
7107 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7108 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7109 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7110 AssertRC(rc);
7111 Log4Func(("Cleared interrupt-window exiting\n"));
7112}
7113
7114
7115/**
7116 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7117 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7118 *
7119 * @param pVCpu The cross context virtual CPU structure.
7120 */
7121DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7122{
7123 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7124 {
7125 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7126 {
7127 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7128 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7129 AssertRC(rc);
7130 Log4Func(("Setup NMI-window exiting\n"));
7131 }
7132 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7133}
7134
7135
7136/**
7137 * Clears the NMI-window exiting control in the VMCS.
7138 *
7139 * @param pVCpu The cross context virtual CPU structure.
7140 */
7141DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7142{
7143 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7144 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7145 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7146 AssertRC(rc);
7147 Log4Func(("Cleared NMI-window exiting\n"));
7148}
7149
7150
7151/**
7152 * Evaluates the event to be delivered to the guest and sets it as the pending
7153 * event.
7154 *
7155 * @returns The VT-x guest-interruptibility state.
7156 * @param pVCpu The cross context virtual CPU structure.
7157 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7158 * out-of-sync. Make sure to update the required fields
7159 * before using them.
7160 */
7161static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7162{
7163 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7164 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7165 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7166 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7167 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7168
7169 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7170 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7171 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7172 Assert(!TRPMHasTrap(pVCpu));
7173
7174 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7175 APICUpdatePendingInterrupts(pVCpu);
7176
7177 /*
7178 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7179 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7180 */
7181 /** @todo SMI. SMIs take priority over NMIs. */
7182 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7183 {
7184 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7185 if ( !pVCpu->hm.s.Event.fPending
7186 && !fBlockNmi
7187 && !fBlockSti
7188 && !fBlockMovSS)
7189 {
7190 Log4Func(("Pending NMI\n"));
7191 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7192 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7193
7194 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7195 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7196 }
7197 else
7198 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7199 }
7200 /*
7201 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7202 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7203 */
7204 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7205 && !pVCpu->hm.s.fSingleInstruction)
7206 {
7207 Assert(!DBGFIsStepping(pVCpu));
7208 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7209 AssertRCReturn(rc, 0);
7210 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7211 if ( !pVCpu->hm.s.Event.fPending
7212 && !fBlockInt
7213 && !fBlockSti
7214 && !fBlockMovSS)
7215 {
7216 uint8_t u8Interrupt;
7217 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7218 if (RT_SUCCESS(rc))
7219 {
7220 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7221 uint32_t u32IntInfo = u8Interrupt
7222 | VMX_EXIT_INTERRUPTION_INFO_VALID
7223 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7224
7225 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7226 }
7227 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7228 {
7229 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7230 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7231 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7232
7233 /*
7234 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7235 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7236 * need to re-set this force-flag here.
7237 */
7238 }
7239 else
7240 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7241 }
7242 else
7243 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7244 }
7245
7246 return fIntrState;
7247}
7248
7249
7250/**
7251 * Sets a pending-debug exception to be delivered to the guest if the guest is
7252 * single-stepping in the VMCS.
7253 *
7254 * @param pVCpu The cross context virtual CPU structure.
7255 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7256 * out-of-sync. Make sure to update the required fields
7257 * before using them.
7258 */
7259DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7260{
7261 RT_NOREF(pVCpu);
7262 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7263 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7264}
7265
7266
7267/**
7268 * Injects any pending events into the guest if the guest is in a state to
7269 * receive them.
7270 *
7271 * @returns Strict VBox status code (i.e. informational status codes too).
7272 * @param pVCpu The cross context virtual CPU structure.
7273 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7274 * out-of-sync. Make sure to update the required fields
7275 * before using them.
7276 * @param fIntrState The VT-x guest-interruptibility state.
7277 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7278 * return VINF_EM_DBG_STEPPED if the event was
7279 * dispatched directly.
7280 */
7281static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7282{
7283 HMVMX_ASSERT_PREEMPT_SAFE();
7284 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7285
7286 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7287 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7288
7289 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7290 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7291 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7292 Assert(!TRPMHasTrap(pVCpu));
7293
7294 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7295 if (pVCpu->hm.s.Event.fPending)
7296 {
7297 /*
7298 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7299 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7300 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7301 *
7302 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7303 */
7304 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7305#ifdef VBOX_STRICT
7306 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7307 {
7308 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7309 Assert(!fBlockInt);
7310 Assert(!fBlockSti);
7311 Assert(!fBlockMovSS);
7312 }
7313 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7314 {
7315 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7316 Assert(!fBlockSti);
7317 Assert(!fBlockMovSS);
7318 Assert(!fBlockNmi);
7319 }
7320#endif
7321 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7322 uIntType));
7323 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7324 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7325 &fIntrState);
7326 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7327
7328 /* Update the interruptibility-state as it could have been changed by
7329 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7330 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7331 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7332
7333 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7334 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7335 else
7336 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7337 }
7338
7339 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7340 if ( fBlockSti
7341 || fBlockMovSS)
7342 {
7343 if (!pVCpu->hm.s.fSingleInstruction)
7344 {
7345 /*
7346 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7347 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7348 * See Intel spec. 27.3.4 "Saving Non-Register State".
7349 */
7350 Assert(!DBGFIsStepping(pVCpu));
7351 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7352 AssertRCReturn(rc, rc);
7353 if (pMixedCtx->eflags.Bits.u1TF)
7354 {
7355 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7356 AssertRCReturn(rc2, rc2);
7357 }
7358 }
7359 else if (pMixedCtx->eflags.Bits.u1TF)
7360 {
7361 /*
7362 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7363 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7364 */
7365 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7366 fIntrState = 0;
7367 }
7368 }
7369
7370 /*
7371 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7372 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7373 */
7374 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7375 AssertRCReturn(rc3, rc3);
7376
7377 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7378 NOREF(fBlockMovSS); NOREF(fBlockSti);
7379 return rcStrict;
7380}
7381
7382
7383/**
7384 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7385 *
7386 * @param pVCpu The cross context virtual CPU structure.
7387 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7388 * out-of-sync. Make sure to update the required fields
7389 * before using them.
7390 */
7391DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7392{
7393 NOREF(pMixedCtx);
7394 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7395 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7396}
7397
7398
7399/**
7400 * Injects a double-fault (\#DF) exception into the VM.
7401 *
7402 * @returns Strict VBox status code (i.e. informational status codes too).
7403 * @param pVCpu The cross context virtual CPU structure.
7404 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7405 * out-of-sync. Make sure to update the required fields
7406 * before using them.
7407 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7408 * and should return VINF_EM_DBG_STEPPED if the event
7409 * is injected directly (register modified by us, not
7410 * by hardware on VM-entry).
7411 * @param pfIntrState Pointer to the current guest interruptibility-state.
7412 * This interruptibility-state will be updated if
7413 * necessary. This cannot not be NULL.
7414 */
7415DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7416{
7417 NOREF(pMixedCtx);
7418 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7419 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7420 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7421 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7422 pfIntrState);
7423}
7424
7425
7426/**
7427 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7428 *
7429 * @param pVCpu The cross context virtual CPU structure.
7430 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7431 * out-of-sync. Make sure to update the required fields
7432 * before using them.
7433 */
7434DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7435{
7436 NOREF(pMixedCtx);
7437 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7438 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7439 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7440}
7441
7442
7443/**
7444 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7445 *
7446 * @param pVCpu The cross context virtual CPU structure.
7447 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7448 * out-of-sync. Make sure to update the required fields
7449 * before using them.
7450 * @param cbInstr The value of RIP that is to be pushed on the guest
7451 * stack.
7452 */
7453DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7454{
7455 NOREF(pMixedCtx);
7456 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7457 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7458 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7459}
7460
7461
7462/**
7463 * Injects a general-protection (\#GP) fault into the VM.
7464 *
7465 * @returns Strict VBox status code (i.e. informational status codes too).
7466 * @param pVCpu The cross context virtual CPU structure.
7467 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7468 * out-of-sync. Make sure to update the required fields
7469 * before using them.
7470 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7471 * mode, i.e. in real-mode it's not valid).
7472 * @param u32ErrorCode The error code associated with the \#GP.
7473 * @param fStepping Whether we're running in
7474 * hmR0VmxRunGuestCodeStep() and should return
7475 * VINF_EM_DBG_STEPPED if the event is injected
7476 * directly (register modified by us, not by
7477 * hardware on VM-entry).
7478 * @param pfIntrState Pointer to the current guest interruptibility-state.
7479 * This interruptibility-state will be updated if
7480 * necessary. This cannot not be NULL.
7481 */
7482DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7483 bool fStepping, uint32_t *pfIntrState)
7484{
7485 NOREF(pMixedCtx);
7486 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7487 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7488 if (fErrorCodeValid)
7489 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7490 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7491 pfIntrState);
7492}
7493
7494
7495#if 0 /* unused */
7496/**
7497 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7498 * VM.
7499 *
7500 * @param pVCpu The cross context virtual CPU structure.
7501 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7502 * out-of-sync. Make sure to update the required fields
7503 * before using them.
7504 * @param u32ErrorCode The error code associated with the \#GP.
7505 */
7506DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7507{
7508 NOREF(pMixedCtx);
7509 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7510 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7511 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7512 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7513}
7514#endif /* unused */
7515
7516
7517/**
7518 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7519 *
7520 * @param pVCpu The cross context virtual CPU structure.
7521 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7522 * out-of-sync. Make sure to update the required fields
7523 * before using them.
7524 * @param uVector The software interrupt vector number.
7525 * @param cbInstr The value of RIP that is to be pushed on the guest
7526 * stack.
7527 */
7528DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7529{
7530 NOREF(pMixedCtx);
7531 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7532 if ( uVector == X86_XCPT_BP
7533 || uVector == X86_XCPT_OF)
7534 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7535 else
7536 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7537 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7538}
7539
7540
7541/**
7542 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7543 * stack.
7544 *
7545 * @returns Strict VBox status code (i.e. informational status codes too).
7546 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7547 * @param pVM The cross context VM structure.
7548 * @param pMixedCtx Pointer to the guest-CPU context.
7549 * @param uValue The value to push to the guest stack.
7550 */
7551DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7552{
7553 /*
7554 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7555 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7556 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7557 */
7558 if (pMixedCtx->sp == 1)
7559 return VINF_EM_RESET;
7560 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7561 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7562 AssertRC(rc);
7563 return rc;
7564}
7565
7566
7567/**
7568 * Injects an event into the guest upon VM-entry by updating the relevant fields
7569 * in the VM-entry area in the VMCS.
7570 *
7571 * @returns Strict VBox status code (i.e. informational status codes too).
7572 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7573 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7574 *
7575 * @param pVCpu The cross context virtual CPU structure.
7576 * @param u64IntInfo The VM-entry interruption-information field.
7577 * @param cbInstr The VM-entry instruction length in bytes (for
7578 * software interrupts, exceptions and privileged
7579 * software exceptions).
7580 * @param u32ErrCode The VM-entry exception error code.
7581 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7582 * @param pfIntrState Pointer to the current guest interruptibility-state.
7583 * This interruptibility-state will be updated if
7584 * necessary. This cannot not be NULL.
7585 * @param fStepping Whether we're running in
7586 * hmR0VmxRunGuestCodeStep() and should return
7587 * VINF_EM_DBG_STEPPED if the event is injected
7588 * directly (register modified by us, not by
7589 * hardware on VM-entry).
7590 */
7591static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7592 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7593{
7594 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7595 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7596 Assert(pfIntrState);
7597
7598 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7599 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7600 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7601 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7602
7603#ifdef VBOX_STRICT
7604 /*
7605 * Validate the error-code-valid bit for hardware exceptions.
7606 * No error codes for exceptions in real-mode.
7607 *
7608 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7609 */
7610 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7611 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7612 {
7613 switch (uVector)
7614 {
7615 case X86_XCPT_PF:
7616 case X86_XCPT_DF:
7617 case X86_XCPT_TS:
7618 case X86_XCPT_NP:
7619 case X86_XCPT_SS:
7620 case X86_XCPT_GP:
7621 case X86_XCPT_AC:
7622 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7623 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7624 RT_FALL_THRU();
7625 default:
7626 break;
7627 }
7628 }
7629#endif
7630
7631 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7632 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7633 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7634
7635 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7636
7637 /*
7638 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7639 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7640 * interrupt handler in the (real-mode) guest.
7641 *
7642 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7643 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7644 */
7645 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7646 {
7647 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7648 {
7649 /*
7650 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7651 * set the deliver-error-code bit.
7652 *
7653 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7654 */
7655 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7656 }
7657 else
7658 {
7659 PVM pVM = pVCpu->CTX_SUFF(pVM);
7660 Assert(PDMVmmDevHeapIsEnabled(pVM));
7661 Assert(pVM->hm.s.vmx.pRealModeTSS);
7662
7663 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7664 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7665 | CPUMCTX_EXTRN_TABLE_MASK
7666 | CPUMCTX_EXTRN_RIP
7667 | CPUMCTX_EXTRN_RSP
7668 | CPUMCTX_EXTRN_RFLAGS);
7669 AssertRCReturn(rc2, rc2);
7670
7671 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7672 size_t const cbIdtEntry = sizeof(X86IDTR16);
7673 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7674 {
7675 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7676 if (uVector == X86_XCPT_DF)
7677 return VINF_EM_RESET;
7678
7679 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7680 if (uVector == X86_XCPT_GP)
7681 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7682
7683 /*
7684 * If we're injecting an event with no valid IDT entry, inject a #GP.
7685 * No error codes for exceptions in real-mode.
7686 *
7687 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7688 */
7689 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7690 pfIntrState);
7691 }
7692
7693 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7694 uint16_t uGuestIp = pMixedCtx->ip;
7695 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7696 {
7697 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7698 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7699 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7700 }
7701 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7702 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7703
7704 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7705 X86IDTR16 IdtEntry;
7706 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7707 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7708 AssertRCReturn(rc2, rc2);
7709
7710 /* Construct the stack frame for the interrupt/exception handler. */
7711 VBOXSTRICTRC rcStrict;
7712 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7713 if (rcStrict == VINF_SUCCESS)
7714 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7715 if (rcStrict == VINF_SUCCESS)
7716 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7717
7718 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7719 if (rcStrict == VINF_SUCCESS)
7720 {
7721 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7722 pMixedCtx->rip = IdtEntry.offSel;
7723 pMixedCtx->cs.Sel = IdtEntry.uSel;
7724 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7725 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7726 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7727 && uVector == X86_XCPT_PF)
7728 pMixedCtx->cr2 = GCPtrFaultAddress;
7729
7730 /* If any other guest-state bits are changed here, make sure to update
7731 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7732 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7733 | HM_CHANGED_GUEST_CR2
7734 | HM_CHANGED_GUEST_RIP
7735 | HM_CHANGED_GUEST_RFLAGS
7736 | HM_CHANGED_GUEST_RSP);
7737
7738 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7739 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7740 {
7741 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7742 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7743 Log4Func(("Clearing inhibition due to STI\n"));
7744 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7745 }
7746 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7747 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7748
7749 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7750 it, if we are returning to ring-3 before executing guest code. */
7751 pVCpu->hm.s.Event.fPending = false;
7752
7753 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7754 if (fStepping)
7755 rcStrict = VINF_EM_DBG_STEPPED;
7756 }
7757 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7758 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7759 return rcStrict;
7760 }
7761 }
7762
7763 /* Validate. */
7764 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7765 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7766 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7767
7768 /* Inject. */
7769 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7770 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7771 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7772 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7773 AssertRCReturn(rc, rc);
7774
7775 /* Update CR2. */
7776 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7777 && uVector == X86_XCPT_PF)
7778 pMixedCtx->cr2 = GCPtrFaultAddress;
7779
7780 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7781
7782 return VINF_SUCCESS;
7783}
7784
7785
7786/**
7787 * Clears the interrupt-window exiting control in the VMCS and if necessary
7788 * clears the current event in the VMCS as well.
7789 *
7790 * @returns VBox status code.
7791 * @param pVCpu The cross context virtual CPU structure.
7792 *
7793 * @remarks Use this function only to clear events that have not yet been
7794 * delivered to the guest but are injected in the VMCS!
7795 * @remarks No-long-jump zone!!!
7796 */
7797static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7798{
7799 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7800 {
7801 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7802 Log4Func(("Cleared interrupt widow\n"));
7803 }
7804
7805 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7806 {
7807 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7808 Log4Func(("Cleared interrupt widow\n"));
7809 }
7810}
7811
7812
7813/**
7814 * Enters the VT-x session.
7815 *
7816 * @returns VBox status code.
7817 * @param pVCpu The cross context virtual CPU structure.
7818 * @param pHostCpu Pointer to the global CPU info struct.
7819 */
7820VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7821{
7822 AssertPtr(pVCpu);
7823 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7824 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7825 RT_NOREF(pHostCpu);
7826
7827 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7828 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7829 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7830
7831#ifdef VBOX_STRICT
7832 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7833 RTCCUINTREG uHostCR4 = ASMGetCR4();
7834 if (!(uHostCR4 & X86_CR4_VMXE))
7835 {
7836 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7837 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7838 }
7839#endif
7840
7841 /*
7842 * Load the VCPU's VMCS as the current (and active) one.
7843 */
7844 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7845 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7846 if (RT_FAILURE(rc))
7847 return rc;
7848
7849 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7850 pVCpu->hm.s.fLeaveDone = false;
7851 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7852
7853 return VINF_SUCCESS;
7854}
7855
7856
7857/**
7858 * The thread-context callback (only on platforms which support it).
7859 *
7860 * @param enmEvent The thread-context event.
7861 * @param pVCpu The cross context virtual CPU structure.
7862 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7863 * @thread EMT(pVCpu)
7864 */
7865VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7866{
7867 NOREF(fGlobalInit);
7868
7869 switch (enmEvent)
7870 {
7871 case RTTHREADCTXEVENT_OUT:
7872 {
7873 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7874 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7875 VMCPU_ASSERT_EMT(pVCpu);
7876
7877 /* No longjmps (logger flushes, locks) in this fragile context. */
7878 VMMRZCallRing3Disable(pVCpu);
7879 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7880
7881 /*
7882 * Restore host-state (FPU, debug etc.)
7883 */
7884 if (!pVCpu->hm.s.fLeaveDone)
7885 {
7886 /*
7887 * Do -not- import the guest-state here as we might already be in the middle of importing
7888 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7889 */
7890 hmR0VmxLeave(pVCpu, false /* fImportState */);
7891 pVCpu->hm.s.fLeaveDone = true;
7892 }
7893
7894 /* Leave HM context, takes care of local init (term). */
7895 int rc = HMR0LeaveCpu(pVCpu);
7896 AssertRC(rc); NOREF(rc);
7897
7898 /* Restore longjmp state. */
7899 VMMRZCallRing3Enable(pVCpu);
7900 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7901 break;
7902 }
7903
7904 case RTTHREADCTXEVENT_IN:
7905 {
7906 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7907 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7908 VMCPU_ASSERT_EMT(pVCpu);
7909
7910 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7911 VMMRZCallRing3Disable(pVCpu);
7912 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7913
7914 /* Initialize the bare minimum state required for HM. This takes care of
7915 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7916 int rc = hmR0EnterCpu(pVCpu);
7917 AssertRC(rc);
7918 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7919 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7920
7921 /* Load the active VMCS as the current one. */
7922 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7923 {
7924 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7925 AssertRC(rc); NOREF(rc);
7926 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7927 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7928 }
7929 pVCpu->hm.s.fLeaveDone = false;
7930
7931 /* Restore longjmp state. */
7932 VMMRZCallRing3Enable(pVCpu);
7933 break;
7934 }
7935
7936 default:
7937 break;
7938 }
7939}
7940
7941
7942/**
7943 * Exports the host state into the VMCS host-state area.
7944 * Sets up the VM-exit MSR-load area.
7945 *
7946 * The CPU state will be loaded from these fields on every successful VM-exit.
7947 *
7948 * @returns VBox status code.
7949 * @param pVCpu The cross context virtual CPU structure.
7950 *
7951 * @remarks No-long-jump zone!!!
7952 */
7953static int hmR0VmxExportHostState(PVMCPU pVCpu)
7954{
7955 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7956
7957 int rc = VINF_SUCCESS;
7958 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7959 {
7960 rc = hmR0VmxExportHostControlRegs();
7961 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7962
7963 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7964 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7965
7966 rc = hmR0VmxExportHostMsrs(pVCpu);
7967 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7968
7969 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7970 }
7971 return rc;
7972}
7973
7974
7975/**
7976 * Saves the host state in the VMCS host-state.
7977 *
7978 * @returns VBox status code.
7979 * @param pVCpu The cross context virtual CPU structure.
7980 *
7981 * @remarks No-long-jump zone!!!
7982 */
7983VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7984{
7985 AssertPtr(pVCpu);
7986 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7987
7988 /*
7989 * Export the host state here while entering HM context.
7990 * When thread-context hooks are used, we might get preempted and have to re-save the host
7991 * state but most of the time we won't be, so do it here before we disable interrupts.
7992 */
7993 return hmR0VmxExportHostState(pVCpu);
7994}
7995
7996
7997/**
7998 * Exports the guest state into the VMCS guest-state area.
7999 *
8000 * The will typically be done before VM-entry when the guest-CPU state and the
8001 * VMCS state may potentially be out of sync.
8002 *
8003 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8004 * VM-entry controls.
8005 * Sets up the appropriate VMX non-root function to execute guest code based on
8006 * the guest CPU mode.
8007 *
8008 * @returns VBox strict status code.
8009 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8010 * without unrestricted guest access and the VMMDev is not presently
8011 * mapped (e.g. EFI32).
8012 *
8013 * @param pVCpu The cross context virtual CPU structure.
8014 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8015 * out-of-sync. Make sure to update the required fields
8016 * before using them.
8017 *
8018 * @remarks No-long-jump zone!!!
8019 */
8020static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8021{
8022 AssertPtr(pVCpu);
8023 AssertPtr(pMixedCtx);
8024 HMVMX_ASSERT_PREEMPT_SAFE();
8025
8026 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8027
8028 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8029
8030 /* Determine real-on-v86 mode. */
8031 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8032 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8033 && CPUMIsGuestInRealModeEx(pMixedCtx))
8034 {
8035 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8036 }
8037
8038 /*
8039 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8040 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8041 */
8042 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8043 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8044
8045 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8046 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8047 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8048
8049 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8050 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8051 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8052
8053 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8054 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8055
8056 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8057 if (rcStrict == VINF_SUCCESS)
8058 { /* likely */ }
8059 else
8060 {
8061 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8062 return rcStrict;
8063 }
8064
8065 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8066 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8067
8068 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8069 may alter controls if we determine we don't have to swap EFER after all. */
8070 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8071 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8072
8073 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8074 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8075
8076 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8077 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8078 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8079
8080 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8081 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8082 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8083 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8084 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8085 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8086
8087 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8088 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8089 | HM_CHANGED_GUEST_CR2
8090 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8091 | HM_CHANGED_GUEST_X87
8092 | HM_CHANGED_GUEST_SSE_AVX
8093 | HM_CHANGED_GUEST_OTHER_XSAVE
8094 | HM_CHANGED_GUEST_XCRx
8095 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8096 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8097 | HM_CHANGED_GUEST_TSC_AUX
8098 | HM_CHANGED_GUEST_OTHER_MSRS
8099 | HM_CHANGED_GUEST_HWVIRT
8100 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8101
8102 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8103 return rc;
8104}
8105
8106
8107/**
8108 * Exports the state shared between the host and guest into the VMCS.
8109 *
8110 * @param pVCpu The cross context virtual CPU structure.
8111 * @param pCtx Pointer to the guest-CPU context.
8112 *
8113 * @remarks No-long-jump zone!!!
8114 */
8115static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8116{
8117 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8118 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8119
8120 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8121 {
8122 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8123 AssertRC(rc);
8124 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8125
8126 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8127 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8128 {
8129 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8130 AssertRC(rc);
8131 }
8132 }
8133
8134 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8135 {
8136 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8137 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8138 }
8139
8140 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8141 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8142}
8143
8144
8145/**
8146 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8147 *
8148 * @returns Strict VBox status code (i.e. informational status codes too).
8149 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8150 * without unrestricted guest access and the VMMDev is not presently
8151 * mapped (e.g. EFI32).
8152 *
8153 * @param pVCpu The cross context virtual CPU structure.
8154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8155 * out-of-sync. Make sure to update the required fields
8156 * before using them.
8157 *
8158 * @remarks No-long-jump zone!!!
8159 */
8160static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8161{
8162 HMVMX_ASSERT_PREEMPT_SAFE();
8163 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8164 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8165
8166#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8167 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8168#endif
8169
8170 /*
8171 * For many exits it's only RIP that changes and hence try to export it first
8172 * without going through a lot of change flag checks.
8173 */
8174 VBOXSTRICTRC rcStrict;
8175 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8176 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8177 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8178 {
8179 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8180 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8181 { /* likely */}
8182 else
8183 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8184 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8185 }
8186 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8187 {
8188 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8189 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8190 { /* likely */}
8191 else
8192 {
8193 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8194 VBOXSTRICTRC_VAL(rcStrict)));
8195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8196 return rcStrict;
8197 }
8198 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8199 }
8200 else
8201 rcStrict = VINF_SUCCESS;
8202
8203#ifdef VBOX_STRICT
8204 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8205 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8206 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8207 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8208 ("fCtxChanged=%#RX64\n", fCtxChanged));
8209#endif
8210 return rcStrict;
8211}
8212
8213
8214/**
8215 * Does the preparations before executing guest code in VT-x.
8216 *
8217 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8218 * recompiler/IEM. We must be cautious what we do here regarding committing
8219 * guest-state information into the VMCS assuming we assuredly execute the
8220 * guest in VT-x mode.
8221 *
8222 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8223 * the common-state (TRPM/forceflags), we must undo those changes so that the
8224 * recompiler/IEM can (and should) use them when it resumes guest execution.
8225 * Otherwise such operations must be done when we can no longer exit to ring-3.
8226 *
8227 * @returns Strict VBox status code (i.e. informational status codes too).
8228 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8229 * have been disabled.
8230 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8231 * double-fault into the guest.
8232 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8233 * dispatched directly.
8234 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8235 *
8236 * @param pVCpu The cross context virtual CPU structure.
8237 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8238 * out-of-sync. Make sure to update the required fields
8239 * before using them.
8240 * @param pVmxTransient Pointer to the VMX transient structure.
8241 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8242 * us ignore some of the reasons for returning to
8243 * ring-3, and return VINF_EM_DBG_STEPPED if event
8244 * dispatching took place.
8245 */
8246static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8247{
8248 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8249
8250#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8251 PGMRZDynMapFlushAutoSet(pVCpu);
8252#endif
8253
8254 /* Check force flag actions that might require us to go back to ring-3. */
8255 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8256 if (rcStrict == VINF_SUCCESS)
8257 { /* FFs doesn't get set all the time. */ }
8258 else
8259 return rcStrict;
8260
8261 /*
8262 * Setup the virtualized-APIC accesses.
8263 *
8264 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8265 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8266 *
8267 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8268 */
8269 PVM pVM = pVCpu->CTX_SUFF(pVM);
8270 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8271 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8272 && PDMHasApic(pVM))
8273 {
8274 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8275 Assert(u64MsrApicBase);
8276 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8277
8278 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8279
8280 /* Unalias any existing mapping. */
8281 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8282 AssertRCReturn(rc, rc);
8283
8284 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8285 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8286 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8287 AssertRCReturn(rc, rc);
8288
8289 /* Update the per-VCPU cache of the APIC base MSR. */
8290 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8291 }
8292
8293 if (TRPMHasTrap(pVCpu))
8294 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8295 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8296
8297 /*
8298 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8299 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8300 * also result in triple-faulting the VM.
8301 */
8302 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8303 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8304 { /* likely */ }
8305 else
8306 {
8307 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8308 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8309 return rcStrict;
8310 }
8311
8312 /*
8313 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8314 * import CR3 themselves. We will need to update them here as even as late as the above
8315 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8316 * the below force flags to be set.
8317 */
8318 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8319 {
8320 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8321 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8322 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8323 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8324 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8325 }
8326 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8327 {
8328 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8329 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8330 }
8331
8332 /*
8333 * No longjmps to ring-3 from this point on!!!
8334 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8335 * This also disables flushing of the R0-logger instance (if any).
8336 */
8337 VMMRZCallRing3Disable(pVCpu);
8338
8339 /*
8340 * Export the guest state bits.
8341 *
8342 * We cannot perform longjmps while loading the guest state because we do not preserve the
8343 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8344 * CPU migration.
8345 *
8346 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8347 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8348 * Hence, loading of the guest state needs to be done -after- injection of events.
8349 */
8350 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8351 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8352 { /* likely */ }
8353 else
8354 {
8355 VMMRZCallRing3Enable(pVCpu);
8356 return rcStrict;
8357 }
8358
8359 /*
8360 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8361 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8362 * preemption disabled for a while. Since this is purly to aid the
8363 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8364 * disable interrupt on NT.
8365 *
8366 * We need to check for force-flags that could've possible been altered since we last
8367 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8368 * see @bugref{6398}).
8369 *
8370 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8371 * to ring-3 before executing guest code.
8372 */
8373 pVmxTransient->fEFlags = ASMIntDisableFlags();
8374
8375 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8376 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8377 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8378 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8379 {
8380 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8381 {
8382 pVCpu->hm.s.Event.fPending = false;
8383
8384 /*
8385 * We've injected any pending events. This is really the point of no return (to ring-3).
8386 *
8387 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8388 * returns from this function, so don't enable them here.
8389 */
8390 return VINF_SUCCESS;
8391 }
8392
8393 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8394 rcStrict = VINF_EM_RAW_INTERRUPT;
8395 }
8396 else
8397 {
8398 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8399 rcStrict = VINF_EM_RAW_TO_R3;
8400 }
8401
8402 ASMSetFlags(pVmxTransient->fEFlags);
8403 VMMRZCallRing3Enable(pVCpu);
8404
8405 return rcStrict;
8406}
8407
8408
8409/**
8410 * Prepares to run guest code in VT-x and we've committed to doing so. This
8411 * means there is no backing out to ring-3 or anywhere else at this
8412 * point.
8413 *
8414 * @param pVCpu The cross context virtual CPU structure.
8415 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8416 * out-of-sync. Make sure to update the required fields
8417 * before using them.
8418 * @param pVmxTransient Pointer to the VMX transient structure.
8419 *
8420 * @remarks Called with preemption disabled.
8421 * @remarks No-long-jump zone!!!
8422 */
8423static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8424{
8425 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8426 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8427 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8428
8429 /*
8430 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8431 */
8432 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8433 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8434
8435 PVM pVM = pVCpu->CTX_SUFF(pVM);
8436 if (!CPUMIsGuestFPUStateActive(pVCpu))
8437 {
8438 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8439 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8440 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8441 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8442 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8443 }
8444
8445 /*
8446 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8447 */
8448 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8449 && pVCpu->hm.s.vmx.cMsrs > 0)
8450 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8451
8452 /*
8453 * Re-save the host state bits as we may've been preempted (only happens when
8454 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8455 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8456 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8457 * See @bugref{8432}.
8458 */
8459 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8460 {
8461 int rc = hmR0VmxExportHostState(pVCpu);
8462 AssertRC(rc);
8463 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8464 }
8465 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8466
8467 /*
8468 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8469 */
8470 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8471 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8472 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8473
8474 /* Store status of the shared guest-host state at the time of VM-entry. */
8475#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8476 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8477 {
8478 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8479 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8480 }
8481 else
8482#endif
8483 {
8484 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8485 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8486 }
8487
8488 /*
8489 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8490 */
8491 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8492 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8493
8494 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8495 RTCPUID idCurrentCpu = pCpu->idCpu;
8496 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8497 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8498 {
8499 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8500 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8501 }
8502
8503 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8504 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8505 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8506 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8507
8508 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8509
8510 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8511 to start executing. */
8512
8513 /*
8514 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8515 */
8516 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8517 {
8518 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8519 {
8520 bool fMsrUpdated;
8521 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8522 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8523 &fMsrUpdated);
8524 AssertRC(rc2);
8525 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8526 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8527 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8528 }
8529 else
8530 {
8531 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8532 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8533 }
8534 }
8535
8536 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8537 {
8538 bool fMsrUpdated;
8539 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8540 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8541 &fMsrUpdated);
8542 AssertRC(rc2);
8543 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8544 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8545 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8546 }
8547
8548#ifdef VBOX_STRICT
8549 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8550 hmR0VmxCheckHostEferMsr(pVCpu);
8551 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8552#endif
8553#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8554 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8555 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8556 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8557#endif
8558}
8559
8560
8561/**
8562 * Performs some essential restoration of state after running guest code in
8563 * VT-x.
8564 *
8565 * @param pVCpu The cross context virtual CPU structure.
8566 * @param pVmxTransient Pointer to the VMX transient structure.
8567 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8568 *
8569 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8570 *
8571 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8572 * unconditionally when it is safe to do so.
8573 */
8574static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8575{
8576 uint64_t const uHostTsc = ASMReadTSC();
8577 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8578
8579 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8580 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8581 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8582 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8583 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8584 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8585
8586 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8587 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8588
8589 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8590 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8591 Assert(!ASMIntAreEnabled());
8592 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8593
8594#if HC_ARCH_BITS == 64
8595 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8596#endif
8597#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8598 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8599 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8600 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8601#else
8602 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8603#endif
8604#ifdef VBOX_STRICT
8605 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8606#endif
8607 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8608
8609 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8610 uint32_t uExitReason;
8611 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8612 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8613 AssertRC(rc);
8614 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8615 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8616
8617 if (rcVMRun == VINF_SUCCESS)
8618 {
8619 /*
8620 * Update the VM-exit history array here even if the VM-entry failed due to:
8621 * - Invalid guest state.
8622 * - MSR loading.
8623 * - Machine-check event.
8624 *
8625 * In any of the above cases we will still have a "valid" VM-exit reason
8626 * despite @a fVMEntryFailed being false.
8627 *
8628 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8629 *
8630 * Note! We don't have CS or RIP at this point. Will probably address that later
8631 * by amending the history entry added here.
8632 */
8633 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8634 UINT64_MAX, uHostTsc);
8635
8636 if (!pVmxTransient->fVMEntryFailed)
8637 {
8638 VMMRZCallRing3Enable(pVCpu);
8639
8640 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8641 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8642
8643#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8644 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8645 AssertRC(rc);
8646#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8647 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8648 AssertRC(rc);
8649#else
8650 /*
8651 * Import the guest-interruptibility state always as we need it while evaluating
8652 * injecting events on re-entry.
8653 *
8654 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8655 * checking for real-mode while exporting the state because all bits that cause
8656 * mode changes wrt CR0 are intercepted.
8657 */
8658 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8659 AssertRC(rc);
8660#endif
8661
8662 /*
8663 * Sync the TPR shadow with our APIC state.
8664 */
8665 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8666 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8667 {
8668 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8669 AssertRC(rc);
8670 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8671 }
8672
8673 return;
8674 }
8675 }
8676 else
8677 {
8678 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8679 }
8680
8681 VMMRZCallRing3Enable(pVCpu);
8682}
8683
8684
8685/**
8686 * Runs the guest code using VT-x the normal way.
8687 *
8688 * @returns VBox status code.
8689 * @param pVCpu The cross context virtual CPU structure.
8690 * @param pCtx Pointer to the guest-CPU context.
8691 *
8692 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8693 */
8694static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8695{
8696 VMXTRANSIENT VmxTransient;
8697 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8698 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8699 uint32_t cLoops = 0;
8700
8701 for (;; cLoops++)
8702 {
8703 Assert(!HMR0SuspendPending());
8704 HMVMX_ASSERT_CPU_SAFE();
8705
8706 /* Preparatory work for running guest code, this may force us to return
8707 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8708 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8709 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8710 if (rcStrict != VINF_SUCCESS)
8711 break;
8712
8713 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8714 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8715 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8716
8717 /* Restore any residual host-state and save any bits shared between host
8718 and guest into the guest-CPU state. Re-enables interrupts! */
8719 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8720
8721 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8722 if (RT_SUCCESS(rcRun))
8723 { /* very likely */ }
8724 else
8725 {
8726 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8727 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8728 return rcRun;
8729 }
8730
8731 /* Profile the VM-exit. */
8732 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8733 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8734 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8735 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8736 HMVMX_START_EXIT_DISPATCH_PROF();
8737
8738 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8739
8740 /* Handle the VM-exit. */
8741#ifdef HMVMX_USE_FUNCTION_TABLE
8742 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8743#else
8744 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8745#endif
8746 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8747 if (rcStrict == VINF_SUCCESS)
8748 {
8749 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8750 continue; /* likely */
8751 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8752 rcStrict = VINF_EM_RAW_INTERRUPT;
8753 }
8754 break;
8755 }
8756
8757 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8758 return rcStrict;
8759}
8760
8761
8762
8763/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8764 * probes.
8765 *
8766 * The following few functions and associated structure contains the bloat
8767 * necessary for providing detailed debug events and dtrace probes as well as
8768 * reliable host side single stepping. This works on the principle of
8769 * "subclassing" the normal execution loop and workers. We replace the loop
8770 * method completely and override selected helpers to add necessary adjustments
8771 * to their core operation.
8772 *
8773 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8774 * any performance for debug and analysis features.
8775 *
8776 * @{
8777 */
8778
8779/**
8780 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8781 * the debug run loop.
8782 */
8783typedef struct VMXRUNDBGSTATE
8784{
8785 /** The RIP we started executing at. This is for detecting that we stepped. */
8786 uint64_t uRipStart;
8787 /** The CS we started executing with. */
8788 uint16_t uCsStart;
8789
8790 /** Whether we've actually modified the 1st execution control field. */
8791 bool fModifiedProcCtls : 1;
8792 /** Whether we've actually modified the 2nd execution control field. */
8793 bool fModifiedProcCtls2 : 1;
8794 /** Whether we've actually modified the exception bitmap. */
8795 bool fModifiedXcptBitmap : 1;
8796
8797 /** We desire the modified the CR0 mask to be cleared. */
8798 bool fClearCr0Mask : 1;
8799 /** We desire the modified the CR4 mask to be cleared. */
8800 bool fClearCr4Mask : 1;
8801 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8802 uint32_t fCpe1Extra;
8803 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8804 uint32_t fCpe1Unwanted;
8805 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8806 uint32_t fCpe2Extra;
8807 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8808 uint32_t bmXcptExtra;
8809 /** The sequence number of the Dtrace provider settings the state was
8810 * configured against. */
8811 uint32_t uDtraceSettingsSeqNo;
8812 /** VM-exits to check (one bit per VM-exit). */
8813 uint32_t bmExitsToCheck[3];
8814
8815 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8816 uint32_t fProcCtlsInitial;
8817 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8818 uint32_t fProcCtls2Initial;
8819 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8820 uint32_t bmXcptInitial;
8821} VMXRUNDBGSTATE;
8822AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8823typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8824
8825
8826/**
8827 * Initializes the VMXRUNDBGSTATE structure.
8828 *
8829 * @param pVCpu The cross context virtual CPU structure of the
8830 * calling EMT.
8831 * @param pCtx The CPU register context to go with @a pVCpu.
8832 * @param pDbgState The structure to initialize.
8833 */
8834static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8835{
8836 pDbgState->uRipStart = pCtx->rip;
8837 pDbgState->uCsStart = pCtx->cs.Sel;
8838
8839 pDbgState->fModifiedProcCtls = false;
8840 pDbgState->fModifiedProcCtls2 = false;
8841 pDbgState->fModifiedXcptBitmap = false;
8842 pDbgState->fClearCr0Mask = false;
8843 pDbgState->fClearCr4Mask = false;
8844 pDbgState->fCpe1Extra = 0;
8845 pDbgState->fCpe1Unwanted = 0;
8846 pDbgState->fCpe2Extra = 0;
8847 pDbgState->bmXcptExtra = 0;
8848 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8849 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8850 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8851}
8852
8853
8854/**
8855 * Updates the VMSC fields with changes requested by @a pDbgState.
8856 *
8857 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8858 * immediately before executing guest code, i.e. when interrupts are disabled.
8859 * We don't check status codes here as we cannot easily assert or return in the
8860 * latter case.
8861 *
8862 * @param pVCpu The cross context virtual CPU structure.
8863 * @param pDbgState The debug state.
8864 */
8865static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8866{
8867 /*
8868 * Ensure desired flags in VMCS control fields are set.
8869 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8870 *
8871 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8872 * there should be no stale data in pCtx at this point.
8873 */
8874 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8875 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8876 {
8877 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8878 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8879 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8880 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8881 pDbgState->fModifiedProcCtls = true;
8882 }
8883
8884 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8885 {
8886 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8887 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8888 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8889 pDbgState->fModifiedProcCtls2 = true;
8890 }
8891
8892 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8893 {
8894 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8895 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8896 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8897 pDbgState->fModifiedXcptBitmap = true;
8898 }
8899
8900 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8901 {
8902 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8903 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8904 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8905 }
8906
8907 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8908 {
8909 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8910 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8911 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8912 }
8913}
8914
8915
8916static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8917{
8918 /*
8919 * Restore VM-exit control settings as we may not reenter this function the
8920 * next time around.
8921 */
8922 /* We reload the initial value, trigger what we can of recalculations the
8923 next time around. From the looks of things, that's all that's required atm. */
8924 if (pDbgState->fModifiedProcCtls)
8925 {
8926 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8927 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8928 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8929 AssertRCReturn(rc2, rc2);
8930 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8931 }
8932
8933 /* We're currently the only ones messing with this one, so just restore the
8934 cached value and reload the field. */
8935 if ( pDbgState->fModifiedProcCtls2
8936 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8937 {
8938 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8939 AssertRCReturn(rc2, rc2);
8940 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8941 }
8942
8943 /* If we've modified the exception bitmap, we restore it and trigger
8944 reloading and partial recalculation the next time around. */
8945 if (pDbgState->fModifiedXcptBitmap)
8946 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8947
8948 return rcStrict;
8949}
8950
8951
8952/**
8953 * Configures VM-exit controls for current DBGF and DTrace settings.
8954 *
8955 * This updates @a pDbgState and the VMCS execution control fields to reflect
8956 * the necessary VM-exits demanded by DBGF and DTrace.
8957 *
8958 * @param pVCpu The cross context virtual CPU structure.
8959 * @param pDbgState The debug state.
8960 * @param pVmxTransient Pointer to the VMX transient structure. May update
8961 * fUpdateTscOffsettingAndPreemptTimer.
8962 */
8963static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8964{
8965 /*
8966 * Take down the dtrace serial number so we can spot changes.
8967 */
8968 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8969 ASMCompilerBarrier();
8970
8971 /*
8972 * We'll rebuild most of the middle block of data members (holding the
8973 * current settings) as we go along here, so start by clearing it all.
8974 */
8975 pDbgState->bmXcptExtra = 0;
8976 pDbgState->fCpe1Extra = 0;
8977 pDbgState->fCpe1Unwanted = 0;
8978 pDbgState->fCpe2Extra = 0;
8979 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8980 pDbgState->bmExitsToCheck[i] = 0;
8981
8982 /*
8983 * Software interrupts (INT XXh) - no idea how to trigger these...
8984 */
8985 PVM pVM = pVCpu->CTX_SUFF(pVM);
8986 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8987 || VBOXVMM_INT_SOFTWARE_ENABLED())
8988 {
8989 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8990 }
8991
8992 /*
8993 * INT3 breakpoints - triggered by #BP exceptions.
8994 */
8995 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8996 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8997
8998 /*
8999 * Exception bitmap and XCPT events+probes.
9000 */
9001 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9002 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9003 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9004
9005 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9006 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9007 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9008 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9009 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9010 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9011 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9012 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9013 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9014 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9015 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9016 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9017 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9018 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9019 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9020 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9021 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9022 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9023
9024 if (pDbgState->bmXcptExtra)
9025 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9026
9027 /*
9028 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9029 *
9030 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9031 * So, when adding/changing/removing please don't forget to update it.
9032 *
9033 * Some of the macros are picking up local variables to save horizontal space,
9034 * (being able to see it in a table is the lesser evil here).
9035 */
9036#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9037 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9038 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9039#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9040 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9041 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9042 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9043 } else do { } while (0)
9044#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9045 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9046 { \
9047 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9048 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9049 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9050 } else do { } while (0)
9051#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9052 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9053 { \
9054 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9055 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9056 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9057 } else do { } while (0)
9058#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9059 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9060 { \
9061 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9062 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9063 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9064 } else do { } while (0)
9065
9066 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9067 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9068 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9069 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9070 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9071
9072 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9073 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9074 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9075 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9076 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9077 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9078 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9079 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9080 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9081 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9082 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9083 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9084 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9085 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9086 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9087 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9088 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9089 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9090 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9091 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9092 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9093 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9094 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9095 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9096 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9097 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9098 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9099 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9100 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9101 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9102 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9104 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9105 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9106 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9107 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9108
9109 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9110 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9111 {
9112 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9113 | CPUMCTX_EXTRN_CR4
9114 | CPUMCTX_EXTRN_APIC_TPR);
9115 AssertRC(rc);
9116
9117#if 0 /** @todo fix me */
9118 pDbgState->fClearCr0Mask = true;
9119 pDbgState->fClearCr4Mask = true;
9120#endif
9121 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9122 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9123 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9124 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9125 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9126 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9127 require clearing here and in the loop if we start using it. */
9128 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9129 }
9130 else
9131 {
9132 if (pDbgState->fClearCr0Mask)
9133 {
9134 pDbgState->fClearCr0Mask = false;
9135 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9136 }
9137 if (pDbgState->fClearCr4Mask)
9138 {
9139 pDbgState->fClearCr4Mask = false;
9140 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9141 }
9142 }
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9145
9146 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9147 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9148 {
9149 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9150 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9151 }
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9154
9155 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9157 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9159 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9161 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9163#if 0 /** @todo too slow, fix handler. */
9164 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9165#endif
9166 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9167
9168 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9169 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9170 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9171 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9172 {
9173 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9174 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9175 }
9176 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9178 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9180
9181 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9182 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9183 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9184 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9185 {
9186 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9187 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9188 }
9189 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9190 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9191 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9192 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9193
9194 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9195 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9196 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9197 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9198 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9199 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9200 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9201 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9202 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9203 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9204 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9205 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9206 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9207 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9208 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9209 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9210 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9211 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9212 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9213 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9214 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9215 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9216
9217#undef IS_EITHER_ENABLED
9218#undef SET_ONLY_XBM_IF_EITHER_EN
9219#undef SET_CPE1_XBM_IF_EITHER_EN
9220#undef SET_CPEU_XBM_IF_EITHER_EN
9221#undef SET_CPE2_XBM_IF_EITHER_EN
9222
9223 /*
9224 * Sanitize the control stuff.
9225 */
9226 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9227 if (pDbgState->fCpe2Extra)
9228 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9229 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9230 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9231 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9232 {
9233 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9234 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9235 }
9236
9237 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9238 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9239 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9240 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9241}
9242
9243
9244/**
9245 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9246 * appropriate.
9247 *
9248 * The caller has checked the VM-exit against the
9249 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9250 * already, so we don't have to do that either.
9251 *
9252 * @returns Strict VBox status code (i.e. informational status codes too).
9253 * @param pVCpu The cross context virtual CPU structure.
9254 * @param pMixedCtx Pointer to the guest-CPU context.
9255 * @param pVmxTransient Pointer to the VMX-transient structure.
9256 * @param uExitReason The VM-exit reason.
9257 *
9258 * @remarks The name of this function is displayed by dtrace, so keep it short
9259 * and to the point. No longer than 33 chars long, please.
9260 */
9261static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9262 uint32_t uExitReason)
9263{
9264 /*
9265 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9266 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9267 *
9268 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9269 * does. Must add/change/remove both places. Same ordering, please.
9270 *
9271 * Added/removed events must also be reflected in the next section
9272 * where we dispatch dtrace events.
9273 */
9274 bool fDtrace1 = false;
9275 bool fDtrace2 = false;
9276 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9277 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9278 uint32_t uEventArg = 0;
9279#define SET_EXIT(a_EventSubName) \
9280 do { \
9281 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9282 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9283 } while (0)
9284#define SET_BOTH(a_EventSubName) \
9285 do { \
9286 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9287 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9288 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9289 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9290 } while (0)
9291 switch (uExitReason)
9292 {
9293 case VMX_EXIT_MTF:
9294 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9295
9296 case VMX_EXIT_XCPT_OR_NMI:
9297 {
9298 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9299 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9300 {
9301 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9302 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9303 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9304 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9305 {
9306 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9307 {
9308 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9309 uEventArg = pVmxTransient->uExitIntErrorCode;
9310 }
9311 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9312 switch (enmEvent1)
9313 {
9314 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9315 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9316 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9317 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9318 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9319 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9320 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9321 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9322 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9323 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9324 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9325 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9326 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9327 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9328 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9329 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9330 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9331 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9332 default: break;
9333 }
9334 }
9335 else
9336 AssertFailed();
9337 break;
9338
9339 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9340 uEventArg = idxVector;
9341 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9342 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9343 break;
9344 }
9345 break;
9346 }
9347
9348 case VMX_EXIT_TRIPLE_FAULT:
9349 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9350 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9351 break;
9352 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9353 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9354 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9355 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9356 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9357
9358 /* Instruction specific VM-exits: */
9359 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9360 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9361 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9362 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9363 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9364 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9365 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9366 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9367 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9368 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9369 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9370 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9371 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9372 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9373 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9374 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9375 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9376 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9377 case VMX_EXIT_MOV_CRX:
9378 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9379 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9380 SET_BOTH(CRX_READ);
9381 else
9382 SET_BOTH(CRX_WRITE);
9383 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9384 break;
9385 case VMX_EXIT_MOV_DRX:
9386 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9387 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9388 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9389 SET_BOTH(DRX_READ);
9390 else
9391 SET_BOTH(DRX_WRITE);
9392 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9393 break;
9394 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9395 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9396 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9397 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9398 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9399 case VMX_EXIT_XDTR_ACCESS:
9400 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9401 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9402 {
9403 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9404 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9405 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9406 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9407 }
9408 break;
9409
9410 case VMX_EXIT_TR_ACCESS:
9411 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9412 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9413 {
9414 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9415 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9416 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9417 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9418 }
9419 break;
9420
9421 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9422 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9423 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9424 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9425 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9426 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9427 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9428 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9429 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9430 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9431 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9432
9433 /* Events that aren't relevant at this point. */
9434 case VMX_EXIT_EXT_INT:
9435 case VMX_EXIT_INT_WINDOW:
9436 case VMX_EXIT_NMI_WINDOW:
9437 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9438 case VMX_EXIT_PREEMPT_TIMER:
9439 case VMX_EXIT_IO_INSTR:
9440 break;
9441
9442 /* Errors and unexpected events. */
9443 case VMX_EXIT_INIT_SIGNAL:
9444 case VMX_EXIT_SIPI:
9445 case VMX_EXIT_IO_SMI:
9446 case VMX_EXIT_SMI:
9447 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9448 case VMX_EXIT_ERR_MSR_LOAD:
9449 case VMX_EXIT_ERR_MACHINE_CHECK:
9450 break;
9451
9452 default:
9453 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9454 break;
9455 }
9456#undef SET_BOTH
9457#undef SET_EXIT
9458
9459 /*
9460 * Dtrace tracepoints go first. We do them here at once so we don't
9461 * have to copy the guest state saving and stuff a few dozen times.
9462 * Down side is that we've got to repeat the switch, though this time
9463 * we use enmEvent since the probes are a subset of what DBGF does.
9464 */
9465 if (fDtrace1 || fDtrace2)
9466 {
9467 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9468 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9469 switch (enmEvent1)
9470 {
9471 /** @todo consider which extra parameters would be helpful for each probe. */
9472 case DBGFEVENT_END: break;
9473 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9475 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9476 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9477 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9478 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9479 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9480 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9481 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9482 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9483 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9484 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9485 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9486 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9491 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9492 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9493 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9498 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9499 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9500 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9503 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9504 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9505 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9506 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9507 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9518 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9536 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9537 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9538 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9539 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9540 }
9541 switch (enmEvent2)
9542 {
9543 /** @todo consider which extra parameters would be helpful for each probe. */
9544 case DBGFEVENT_END: break;
9545 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9547 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9555 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9556 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9557 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9558 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9559 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9560 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9561 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9597 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9598 }
9599 }
9600
9601 /*
9602 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9603 * the DBGF call will do a full check).
9604 *
9605 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9606 * Note! If we have to events, we prioritize the first, i.e. the instruction
9607 * one, in order to avoid event nesting.
9608 */
9609 PVM pVM = pVCpu->CTX_SUFF(pVM);
9610 if ( enmEvent1 != DBGFEVENT_END
9611 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9612 {
9613 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9614 if (rcStrict != VINF_SUCCESS)
9615 return rcStrict;
9616 }
9617 else if ( enmEvent2 != DBGFEVENT_END
9618 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9619 {
9620 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9621 if (rcStrict != VINF_SUCCESS)
9622 return rcStrict;
9623 }
9624
9625 return VINF_SUCCESS;
9626}
9627
9628
9629/**
9630 * Single-stepping VM-exit filtering.
9631 *
9632 * This is preprocessing the VM-exits and deciding whether we've gotten far
9633 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9634 * handling is performed.
9635 *
9636 * @returns Strict VBox status code (i.e. informational status codes too).
9637 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9638 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9639 * out-of-sync. Make sure to update the required
9640 * fields before using them.
9641 * @param pVmxTransient Pointer to the VMX-transient structure.
9642 * @param uExitReason The VM-exit reason.
9643 * @param pDbgState The debug state.
9644 */
9645DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9646 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9647{
9648 /*
9649 * Expensive (saves context) generic dtrace VM-exit probe.
9650 */
9651 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9652 { /* more likely */ }
9653 else
9654 {
9655 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9656 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9657 AssertRC(rc);
9658 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9659 }
9660
9661 /*
9662 * Check for host NMI, just to get that out of the way.
9663 */
9664 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9665 { /* normally likely */ }
9666 else
9667 {
9668 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9669 AssertRCReturn(rc2, rc2);
9670 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9671 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9672 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9673 }
9674
9675 /*
9676 * Check for single stepping event if we're stepping.
9677 */
9678 if (pVCpu->hm.s.fSingleInstruction)
9679 {
9680 switch (uExitReason)
9681 {
9682 case VMX_EXIT_MTF:
9683 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9684
9685 /* Various events: */
9686 case VMX_EXIT_XCPT_OR_NMI:
9687 case VMX_EXIT_EXT_INT:
9688 case VMX_EXIT_TRIPLE_FAULT:
9689 case VMX_EXIT_INT_WINDOW:
9690 case VMX_EXIT_NMI_WINDOW:
9691 case VMX_EXIT_TASK_SWITCH:
9692 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9693 case VMX_EXIT_APIC_ACCESS:
9694 case VMX_EXIT_EPT_VIOLATION:
9695 case VMX_EXIT_EPT_MISCONFIG:
9696 case VMX_EXIT_PREEMPT_TIMER:
9697
9698 /* Instruction specific VM-exits: */
9699 case VMX_EXIT_CPUID:
9700 case VMX_EXIT_GETSEC:
9701 case VMX_EXIT_HLT:
9702 case VMX_EXIT_INVD:
9703 case VMX_EXIT_INVLPG:
9704 case VMX_EXIT_RDPMC:
9705 case VMX_EXIT_RDTSC:
9706 case VMX_EXIT_RSM:
9707 case VMX_EXIT_VMCALL:
9708 case VMX_EXIT_VMCLEAR:
9709 case VMX_EXIT_VMLAUNCH:
9710 case VMX_EXIT_VMPTRLD:
9711 case VMX_EXIT_VMPTRST:
9712 case VMX_EXIT_VMREAD:
9713 case VMX_EXIT_VMRESUME:
9714 case VMX_EXIT_VMWRITE:
9715 case VMX_EXIT_VMXOFF:
9716 case VMX_EXIT_VMXON:
9717 case VMX_EXIT_MOV_CRX:
9718 case VMX_EXIT_MOV_DRX:
9719 case VMX_EXIT_IO_INSTR:
9720 case VMX_EXIT_RDMSR:
9721 case VMX_EXIT_WRMSR:
9722 case VMX_EXIT_MWAIT:
9723 case VMX_EXIT_MONITOR:
9724 case VMX_EXIT_PAUSE:
9725 case VMX_EXIT_XDTR_ACCESS:
9726 case VMX_EXIT_TR_ACCESS:
9727 case VMX_EXIT_INVEPT:
9728 case VMX_EXIT_RDTSCP:
9729 case VMX_EXIT_INVVPID:
9730 case VMX_EXIT_WBINVD:
9731 case VMX_EXIT_XSETBV:
9732 case VMX_EXIT_RDRAND:
9733 case VMX_EXIT_INVPCID:
9734 case VMX_EXIT_VMFUNC:
9735 case VMX_EXIT_RDSEED:
9736 case VMX_EXIT_XSAVES:
9737 case VMX_EXIT_XRSTORS:
9738 {
9739 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9740 | CPUMCTX_EXTRN_CS);
9741 AssertRCReturn(rc, rc);
9742 if ( pMixedCtx->rip != pDbgState->uRipStart
9743 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9744 return VINF_EM_DBG_STEPPED;
9745 break;
9746 }
9747
9748 /* Errors and unexpected events: */
9749 case VMX_EXIT_INIT_SIGNAL:
9750 case VMX_EXIT_SIPI:
9751 case VMX_EXIT_IO_SMI:
9752 case VMX_EXIT_SMI:
9753 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9754 case VMX_EXIT_ERR_MSR_LOAD:
9755 case VMX_EXIT_ERR_MACHINE_CHECK:
9756 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9757 break;
9758
9759 default:
9760 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9761 break;
9762 }
9763 }
9764
9765 /*
9766 * Check for debugger event breakpoints and dtrace probes.
9767 */
9768 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9769 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9770 {
9771 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9772 if (rcStrict != VINF_SUCCESS)
9773 return rcStrict;
9774 }
9775
9776 /*
9777 * Normal processing.
9778 */
9779#ifdef HMVMX_USE_FUNCTION_TABLE
9780 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9781#else
9782 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9783#endif
9784}
9785
9786
9787/**
9788 * Single steps guest code using VT-x.
9789 *
9790 * @returns Strict VBox status code (i.e. informational status codes too).
9791 * @param pVCpu The cross context virtual CPU structure.
9792 * @param pCtx Pointer to the guest-CPU context.
9793 *
9794 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9795 */
9796static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9797{
9798 VMXTRANSIENT VmxTransient;
9799 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9800
9801 /* Set HMCPU indicators. */
9802 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9803 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9804 pVCpu->hm.s.fDebugWantRdTscExit = false;
9805 pVCpu->hm.s.fUsingDebugLoop = true;
9806
9807 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9808 VMXRUNDBGSTATE DbgState;
9809 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9810 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9811
9812 /*
9813 * The loop.
9814 */
9815 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9816 for (uint32_t cLoops = 0; ; cLoops++)
9817 {
9818 Assert(!HMR0SuspendPending());
9819 HMVMX_ASSERT_CPU_SAFE();
9820 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9821
9822 /*
9823 * Preparatory work for running guest code, this may force us to return
9824 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9825 */
9826 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9827 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9828 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9829 if (rcStrict != VINF_SUCCESS)
9830 break;
9831
9832 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9833 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9834
9835 /*
9836 * Now we can run the guest code.
9837 */
9838 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9839
9840 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9841
9842 /*
9843 * Restore any residual host-state and save any bits shared between host
9844 * and guest into the guest-CPU state. Re-enables interrupts!
9845 */
9846 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9847
9848 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9849 if (RT_SUCCESS(rcRun))
9850 { /* very likely */ }
9851 else
9852 {
9853 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9854 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9855 return rcRun;
9856 }
9857
9858 /* Profile the VM-exit. */
9859 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9861 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9862 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9863 HMVMX_START_EXIT_DISPATCH_PROF();
9864
9865 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9866
9867 /*
9868 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9869 */
9870 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9871 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9872 if (rcStrict != VINF_SUCCESS)
9873 break;
9874 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9875 {
9876 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9877 rcStrict = VINF_EM_RAW_INTERRUPT;
9878 break;
9879 }
9880
9881 /*
9882 * Stepping: Did the RIP change, if so, consider it a single step.
9883 * Otherwise, make sure one of the TFs gets set.
9884 */
9885 if (fStepping)
9886 {
9887 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9888 | CPUMCTX_EXTRN_CS);
9889 AssertRC(rc);
9890 if ( pCtx->rip != DbgState.uRipStart
9891 || pCtx->cs.Sel != DbgState.uCsStart)
9892 {
9893 rcStrict = VINF_EM_DBG_STEPPED;
9894 break;
9895 }
9896 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9897 }
9898
9899 /*
9900 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9901 */
9902 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9903 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9904 }
9905
9906 /*
9907 * Clear the X86_EFL_TF if necessary.
9908 */
9909 if (pVCpu->hm.s.fClearTrapFlag)
9910 {
9911 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9912 AssertRC(rc);
9913 pVCpu->hm.s.fClearTrapFlag = false;
9914 pCtx->eflags.Bits.u1TF = 0;
9915 }
9916 /** @todo there seems to be issues with the resume flag when the monitor trap
9917 * flag is pending without being used. Seen early in bios init when
9918 * accessing APIC page in protected mode. */
9919
9920 /*
9921 * Restore VM-exit control settings as we may not reenter this function the
9922 * next time around.
9923 */
9924 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9925
9926 /* Restore HMCPU indicators. */
9927 pVCpu->hm.s.fUsingDebugLoop = false;
9928 pVCpu->hm.s.fDebugWantRdTscExit = false;
9929 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9930
9931 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9932 return rcStrict;
9933}
9934
9935
9936/** @} */
9937
9938
9939/**
9940 * Checks if any expensive dtrace probes are enabled and we should go to the
9941 * debug loop.
9942 *
9943 * @returns true if we should use debug loop, false if not.
9944 */
9945static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9946{
9947 /* It's probably faster to OR the raw 32-bit counter variables together.
9948 Since the variables are in an array and the probes are next to one
9949 another (more or less), we have good locality. So, better read
9950 eight-nine cache lines ever time and only have one conditional, than
9951 128+ conditionals, right? */
9952 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9953 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9954 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9955 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9956 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9957 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9958 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9959 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9960 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9961 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9962 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9963 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9964 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9965 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9966 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9967 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9968 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9969 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9970 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9971 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9972 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9973 ) != 0
9974 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9975 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9976 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9977 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9978 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9979 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9980 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9981 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9982 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9983 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9984 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9985 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9986 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9987 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9988 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9989 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9990 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9991 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9992 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9993 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9994 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9995 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9996 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9997 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9998 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9999 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10000 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10001 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10002 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10003 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10004 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10005 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10006 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10007 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10008 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10009 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10010 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10011 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10012 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10013 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10014 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10015 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10016 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10017 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10018 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10019 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10020 ) != 0
10021 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10022 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10023 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10024 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10025 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10026 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10027 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10028 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10029 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10030 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10031 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10032 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10033 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10034 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10035 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10036 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10037 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10038 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10039 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10040 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10041 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10042 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10043 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10044 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10045 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10046 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10047 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10048 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10049 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10050 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10051 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10052 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10053 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10054 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10055 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10056 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10057 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10058 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10059 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10060 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10061 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10062 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10063 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10064 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10065 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10066 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10067 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10068 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10069 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10070 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10071 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10072 ) != 0;
10073}
10074
10075
10076/**
10077 * Runs the guest code using VT-x.
10078 *
10079 * @returns Strict VBox status code (i.e. informational status codes too).
10080 * @param pVCpu The cross context virtual CPU structure.
10081 * @param pCtx Pointer to the guest-CPU context.
10082 */
10083VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10084{
10085 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10086 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10087 HMVMX_ASSERT_PREEMPT_SAFE();
10088
10089 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10090
10091 VBOXSTRICTRC rcStrict;
10092 if ( !pVCpu->hm.s.fUseDebugLoop
10093 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10094 && !DBGFIsStepping(pVCpu)
10095 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10096 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10097 else
10098 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10099
10100 if (rcStrict == VERR_EM_INTERPRETER)
10101 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10102 else if (rcStrict == VINF_EM_RESET)
10103 rcStrict = VINF_EM_TRIPLE_FAULT;
10104
10105 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10106 if (RT_FAILURE(rc2))
10107 {
10108 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10109 rcStrict = rc2;
10110 }
10111 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10112 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10113 return rcStrict;
10114}
10115
10116
10117#ifndef HMVMX_USE_FUNCTION_TABLE
10118DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10119{
10120#ifdef DEBUG_ramshankar
10121#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10122 do { \
10123 if (a_fSave != 0) \
10124 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10125 VBOXSTRICTRC rcStrict = a_CallExpr; \
10126 if (a_fSave != 0) \
10127 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10128 return rcStrict; \
10129 } while (0)
10130#else
10131# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10132#endif
10133 switch (rcReason)
10134 {
10135 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10141 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10142 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10143 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10144 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10145 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10146 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10147 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10148 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10149 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10150 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10151 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10152 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10153 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10154 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10155 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10156 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10157 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10158 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10159 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10160 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10161 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10162 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10163 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10164 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10165 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10166 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10167 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10168 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10169
10170 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10171 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10172 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10173 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10174 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10175 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10176 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10177 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10178 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10179
10180 case VMX_EXIT_VMCLEAR:
10181 case VMX_EXIT_VMLAUNCH:
10182 case VMX_EXIT_VMPTRLD:
10183 case VMX_EXIT_VMPTRST:
10184 case VMX_EXIT_VMREAD:
10185 case VMX_EXIT_VMRESUME:
10186 case VMX_EXIT_VMWRITE:
10187 case VMX_EXIT_VMXOFF:
10188 case VMX_EXIT_VMXON:
10189 case VMX_EXIT_INVEPT:
10190 case VMX_EXIT_INVVPID:
10191 case VMX_EXIT_VMFUNC:
10192 case VMX_EXIT_XSAVES:
10193 case VMX_EXIT_XRSTORS:
10194 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10195
10196 case VMX_EXIT_ENCLS:
10197 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10198 case VMX_EXIT_PML_FULL:
10199 default:
10200 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10201 }
10202#undef VMEXIT_CALL_RET
10203}
10204#endif /* !HMVMX_USE_FUNCTION_TABLE */
10205
10206
10207#ifdef VBOX_STRICT
10208/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10209# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10210 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10211
10212# define HMVMX_ASSERT_PREEMPT_CPUID() \
10213 do { \
10214 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10215 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10216 } while (0)
10217
10218# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10219 do { \
10220 AssertPtr(pVCpu); \
10221 AssertPtr(pMixedCtx); \
10222 AssertPtr(pVmxTransient); \
10223 Assert(pVmxTransient->fVMEntryFailed == false); \
10224 Assert(ASMIntAreEnabled()); \
10225 HMVMX_ASSERT_PREEMPT_SAFE(); \
10226 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10227 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10228 HMVMX_ASSERT_PREEMPT_SAFE(); \
10229 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10230 HMVMX_ASSERT_PREEMPT_CPUID(); \
10231 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10232 } while (0)
10233
10234# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10235 do { \
10236 Log4Func(("\n")); \
10237 } while (0)
10238#else /* nonstrict builds: */
10239# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10240 do { \
10241 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10242 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10243 } while (0)
10244# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10245#endif
10246
10247
10248/**
10249 * Advances the guest RIP by the specified number of bytes.
10250 *
10251 * @param pVCpu The cross context virtual CPU structure.
10252 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10253 * out-of-sync. Make sure to update the required fields
10254 * before using them.
10255 * @param cbInstr Number of bytes to advance the RIP by.
10256 *
10257 * @remarks No-long-jump zone!!!
10258 */
10259DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10260{
10261 /* Advance the RIP. */
10262 pMixedCtx->rip += cbInstr;
10263 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10264
10265 /* Update interrupt inhibition. */
10266 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10267 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10268 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10269}
10270
10271
10272/**
10273 * Advances the guest RIP after reading it from the VMCS.
10274 *
10275 * @returns VBox status code, no informational status codes.
10276 * @param pVCpu The cross context virtual CPU structure.
10277 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10278 * out-of-sync. Make sure to update the required fields
10279 * before using them.
10280 * @param pVmxTransient Pointer to the VMX transient structure.
10281 *
10282 * @remarks No-long-jump zone!!!
10283 */
10284static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10285{
10286 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10287 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10288 | CPUMCTX_EXTRN_RFLAGS);
10289 AssertRCReturn(rc, rc);
10290
10291 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10292
10293 /*
10294 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10295 * pending debug exception field as it takes care of priority of events.
10296 *
10297 * See Intel spec. 32.2.1 "Debug Exceptions".
10298 */
10299 if ( !pVCpu->hm.s.fSingleInstruction
10300 && pMixedCtx->eflags.Bits.u1TF)
10301 {
10302 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10303 AssertRCReturn(rc, rc);
10304 }
10305
10306 return VINF_SUCCESS;
10307}
10308
10309
10310/**
10311 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10312 * and update error record fields accordingly.
10313 *
10314 * @return VMX_IGS_* return codes.
10315 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10316 * wrong with the guest state.
10317 *
10318 * @param pVCpu The cross context virtual CPU structure.
10319 * @param pCtx Pointer to the guest-CPU state.
10320 *
10321 * @remarks This function assumes our cache of the VMCS controls
10322 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10323 */
10324static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10325{
10326#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10327#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10328 uError = (err); \
10329 break; \
10330 } else do { } while (0)
10331
10332 int rc;
10333 PVM pVM = pVCpu->CTX_SUFF(pVM);
10334 uint32_t uError = VMX_IGS_ERROR;
10335 uint32_t u32Val;
10336 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10337
10338 do
10339 {
10340 /*
10341 * CR0.
10342 */
10343 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10344 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10345 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10346 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10347 if (fUnrestrictedGuest)
10348 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10349
10350 uint32_t u32GuestCr0;
10351 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10352 AssertRCBreak(rc);
10353 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10354 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10355 if ( !fUnrestrictedGuest
10356 && (u32GuestCr0 & X86_CR0_PG)
10357 && !(u32GuestCr0 & X86_CR0_PE))
10358 {
10359 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10360 }
10361
10362 /*
10363 * CR4.
10364 */
10365 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10366 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10367
10368 uint32_t u32GuestCr4;
10369 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10370 AssertRCBreak(rc);
10371 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10372 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10373
10374 /*
10375 * IA32_DEBUGCTL MSR.
10376 */
10377 uint64_t u64Val;
10378 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10379 AssertRCBreak(rc);
10380 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10381 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10382 {
10383 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10384 }
10385 uint64_t u64DebugCtlMsr = u64Val;
10386
10387#ifdef VBOX_STRICT
10388 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10389 AssertRCBreak(rc);
10390 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10391#endif
10392 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10393
10394 /*
10395 * RIP and RFLAGS.
10396 */
10397 uint32_t u32Eflags;
10398#if HC_ARCH_BITS == 64
10399 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10400 AssertRCBreak(rc);
10401 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10402 if ( !fLongModeGuest
10403 || !pCtx->cs.Attr.n.u1Long)
10404 {
10405 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10406 }
10407 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10408 * must be identical if the "IA-32e mode guest" VM-entry
10409 * control is 1 and CS.L is 1. No check applies if the
10410 * CPU supports 64 linear-address bits. */
10411
10412 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10413 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10414 AssertRCBreak(rc);
10415 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10416 VMX_IGS_RFLAGS_RESERVED);
10417 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10418 u32Eflags = u64Val;
10419#else
10420 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10421 AssertRCBreak(rc);
10422 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10423 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10424#endif
10425
10426 if ( fLongModeGuest
10427 || ( fUnrestrictedGuest
10428 && !(u32GuestCr0 & X86_CR0_PE)))
10429 {
10430 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10431 }
10432
10433 uint32_t u32EntryInfo;
10434 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10435 AssertRCBreak(rc);
10436 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10437 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10438 {
10439 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10440 }
10441
10442 /*
10443 * 64-bit checks.
10444 */
10445#if HC_ARCH_BITS == 64
10446 if (fLongModeGuest)
10447 {
10448 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10449 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10450 }
10451
10452 if ( !fLongModeGuest
10453 && (u32GuestCr4 & X86_CR4_PCIDE))
10454 {
10455 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10456 }
10457
10458 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10459 * 51:32 beyond the processor's physical-address width are 0. */
10460
10461 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10462 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10463 {
10464 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10465 }
10466
10467 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10468 AssertRCBreak(rc);
10469 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10470
10471 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10472 AssertRCBreak(rc);
10473 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10474#endif
10475
10476 /*
10477 * PERF_GLOBAL MSR.
10478 */
10479 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10480 {
10481 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10482 AssertRCBreak(rc);
10483 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10484 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10485 }
10486
10487 /*
10488 * PAT MSR.
10489 */
10490 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10491 {
10492 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10493 AssertRCBreak(rc);
10494 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10495 for (unsigned i = 0; i < 8; i++)
10496 {
10497 uint8_t u8Val = (u64Val & 0xff);
10498 if ( u8Val != 0 /* UC */
10499 && u8Val != 1 /* WC */
10500 && u8Val != 4 /* WT */
10501 && u8Val != 5 /* WP */
10502 && u8Val != 6 /* WB */
10503 && u8Val != 7 /* UC- */)
10504 {
10505 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10506 }
10507 u64Val >>= 8;
10508 }
10509 }
10510
10511 /*
10512 * EFER MSR.
10513 */
10514 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10515 {
10516 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10517 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10518 AssertRCBreak(rc);
10519 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10520 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10521 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10522 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10523 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10524 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10525 || !(u32GuestCr0 & X86_CR0_PG)
10526 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10527 VMX_IGS_EFER_LMA_LME_MISMATCH);
10528 }
10529
10530 /*
10531 * Segment registers.
10532 */
10533 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10534 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10535 if (!(u32Eflags & X86_EFL_VM))
10536 {
10537 /* CS */
10538 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10539 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10540 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10541 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10542 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10543 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10544 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10545 /* CS cannot be loaded with NULL in protected mode. */
10546 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10547 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10548 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10549 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10550 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10551 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10552 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10553 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10554 else
10555 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10556
10557 /* SS */
10558 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10559 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10560 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10561 if ( !(pCtx->cr0 & X86_CR0_PE)
10562 || pCtx->cs.Attr.n.u4Type == 3)
10563 {
10564 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10565 }
10566 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10567 {
10568 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10569 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10570 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10571 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10572 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10573 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10574 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10575 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10576 }
10577
10578 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10579 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10580 {
10581 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10582 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10583 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10584 || pCtx->ds.Attr.n.u4Type > 11
10585 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10586 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10587 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10588 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10589 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10590 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10591 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10592 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10593 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10594 }
10595 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10596 {
10597 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10598 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10599 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10600 || pCtx->es.Attr.n.u4Type > 11
10601 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10602 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10603 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10604 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10605 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10606 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10607 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10608 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10609 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10610 }
10611 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10612 {
10613 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10614 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10615 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10616 || pCtx->fs.Attr.n.u4Type > 11
10617 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10618 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10619 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10620 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10621 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10622 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10623 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10624 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10625 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10626 }
10627 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10628 {
10629 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10630 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10631 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10632 || pCtx->gs.Attr.n.u4Type > 11
10633 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10634 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10635 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10636 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10637 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10638 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10639 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10640 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10641 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10642 }
10643 /* 64-bit capable CPUs. */
10644#if HC_ARCH_BITS == 64
10645 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10646 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10647 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10648 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10649 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10650 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10651 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10652 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10653 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10654 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10655 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10656#endif
10657 }
10658 else
10659 {
10660 /* V86 mode checks. */
10661 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10662 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10663 {
10664 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10665 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10666 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10667 }
10668 else
10669 {
10670 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10671 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10672 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10673 }
10674
10675 /* CS */
10676 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10677 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10678 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10679 /* SS */
10680 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10681 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10682 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10683 /* DS */
10684 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10685 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10686 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10687 /* ES */
10688 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10689 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10690 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10691 /* FS */
10692 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10693 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10694 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10695 /* GS */
10696 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10697 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10698 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10699 /* 64-bit capable CPUs. */
10700#if HC_ARCH_BITS == 64
10701 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10702 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10703 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10704 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10705 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10706 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10707 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10708 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10709 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10710 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10711 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10712#endif
10713 }
10714
10715 /*
10716 * TR.
10717 */
10718 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10719 /* 64-bit capable CPUs. */
10720#if HC_ARCH_BITS == 64
10721 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10722#endif
10723 if (fLongModeGuest)
10724 {
10725 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10726 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10727 }
10728 else
10729 {
10730 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10731 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10732 VMX_IGS_TR_ATTR_TYPE_INVALID);
10733 }
10734 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10735 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10736 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10737 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10738 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10739 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10740 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10741 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10742
10743 /*
10744 * GDTR and IDTR.
10745 */
10746#if HC_ARCH_BITS == 64
10747 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10748 AssertRCBreak(rc);
10749 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10750
10751 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10752 AssertRCBreak(rc);
10753 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10754#endif
10755
10756 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10757 AssertRCBreak(rc);
10758 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10759
10760 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10761 AssertRCBreak(rc);
10762 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10763
10764 /*
10765 * Guest Non-Register State.
10766 */
10767 /* Activity State. */
10768 uint32_t u32ActivityState;
10769 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10770 AssertRCBreak(rc);
10771 HMVMX_CHECK_BREAK( !u32ActivityState
10772 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10773 VMX_IGS_ACTIVITY_STATE_INVALID);
10774 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10775 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10776 uint32_t u32IntrState;
10777 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10778 AssertRCBreak(rc);
10779 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10780 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10781 {
10782 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10783 }
10784
10785 /** @todo Activity state and injecting interrupts. Left as a todo since we
10786 * currently don't use activity states but ACTIVE. */
10787
10788 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10789 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10790
10791 /* Guest interruptibility-state. */
10792 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10793 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10794 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10795 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10796 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10797 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10798 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10799 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10800 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10801 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10802 {
10803 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10804 {
10805 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10806 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10807 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10808 }
10809 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10810 {
10811 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10812 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10813 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10814 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10815 }
10816 }
10817 /** @todo Assumes the processor is not in SMM. */
10818 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10819 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10820 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10821 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10822 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10823 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10824 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10825 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10826 {
10827 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10828 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10829 }
10830
10831 /* Pending debug exceptions. */
10832#if HC_ARCH_BITS == 64
10833 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10834 AssertRCBreak(rc);
10835 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10836 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10837 u32Val = u64Val; /* For pending debug exceptions checks below. */
10838#else
10839 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10840 AssertRCBreak(rc);
10841 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10842 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10843#endif
10844
10845 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10846 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10847 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10848 {
10849 if ( (u32Eflags & X86_EFL_TF)
10850 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10851 {
10852 /* Bit 14 is PendingDebug.BS. */
10853 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10854 }
10855 if ( !(u32Eflags & X86_EFL_TF)
10856 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10857 {
10858 /* Bit 14 is PendingDebug.BS. */
10859 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10860 }
10861 }
10862
10863 /* VMCS link pointer. */
10864 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10865 AssertRCBreak(rc);
10866 if (u64Val != UINT64_C(0xffffffffffffffff))
10867 {
10868 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10869 /** @todo Bits beyond the processor's physical-address width MBZ. */
10870 /** @todo 32-bit located in memory referenced by value of this field (as a
10871 * physical address) must contain the processor's VMCS revision ID. */
10872 /** @todo SMM checks. */
10873 }
10874
10875 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10876 * not using Nested Paging? */
10877 if ( pVM->hm.s.fNestedPaging
10878 && !fLongModeGuest
10879 && CPUMIsGuestInPAEModeEx(pCtx))
10880 {
10881 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10882 AssertRCBreak(rc);
10883 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10884
10885 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10886 AssertRCBreak(rc);
10887 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10888
10889 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10890 AssertRCBreak(rc);
10891 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10892
10893 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10894 AssertRCBreak(rc);
10895 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10896 }
10897
10898 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10899 if (uError == VMX_IGS_ERROR)
10900 uError = VMX_IGS_REASON_NOT_FOUND;
10901 } while (0);
10902
10903 pVCpu->hm.s.u32HMError = uError;
10904 return uError;
10905
10906#undef HMVMX_ERROR_BREAK
10907#undef HMVMX_CHECK_BREAK
10908}
10909
10910/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10911/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10912/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10913
10914/** @name VM-exit handlers.
10915 * @{
10916 */
10917
10918/**
10919 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10920 */
10921HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10922{
10923 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10924 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10925 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10926 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10927 return VINF_SUCCESS;
10928 return VINF_EM_RAW_INTERRUPT;
10929}
10930
10931
10932/**
10933 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10934 */
10935HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10936{
10937 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10938 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10939
10940 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10941 AssertRCReturn(rc, rc);
10942
10943 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10944 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10945 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10946 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10947
10948 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10949 {
10950 /*
10951 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10952 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10953 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10954 *
10955 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10956 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10957 */
10958 VMXDispatchHostNmi();
10959 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10960 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10961 return VINF_SUCCESS;
10962 }
10963
10964 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10965 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10966 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10967 { /* likely */ }
10968 else
10969 {
10970 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10971 rcStrictRc1 = VINF_SUCCESS;
10972 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10973 return rcStrictRc1;
10974 }
10975
10976 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10977 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10978 switch (uIntType)
10979 {
10980 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10981 Assert(uVector == X86_XCPT_DB);
10982 RT_FALL_THRU();
10983 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10984 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10985 RT_FALL_THRU();
10986 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10987 {
10988 /*
10989 * If there's any exception caused as a result of event injection, the resulting
10990 * secondary/final execption will be pending, we shall continue guest execution
10991 * after injecting the event. The page-fault case is complicated and we manually
10992 * handle any currently pending event in hmR0VmxExitXcptPF.
10993 */
10994 if (!pVCpu->hm.s.Event.fPending)
10995 { /* likely */ }
10996 else if (uVector != X86_XCPT_PF)
10997 {
10998 rc = VINF_SUCCESS;
10999 break;
11000 }
11001
11002 switch (uVector)
11003 {
11004 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11005 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11006 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11007 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11008 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11009 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11010
11011 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11012 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11013 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11014 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11015 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11016 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11017 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11018 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11019 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11020 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11021 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11022 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11023 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11024 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11025 default:
11026 {
11027 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11028 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11029 {
11030 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11031 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11032 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11033
11034 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11035 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11036 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11037 AssertRCReturn(rc, rc);
11038 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11039 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11040 0 /* GCPtrFaultAddress */);
11041 }
11042 else
11043 {
11044 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11045 pVCpu->hm.s.u32HMError = uVector;
11046 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11047 }
11048 break;
11049 }
11050 }
11051 break;
11052 }
11053
11054 default:
11055 {
11056 pVCpu->hm.s.u32HMError = uExitIntInfo;
11057 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11058 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11059 break;
11060 }
11061 }
11062 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11063 return rc;
11064}
11065
11066
11067/**
11068 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11069 */
11070HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11071{
11072 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11073
11074 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11075 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11076
11077 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11079 return VINF_SUCCESS;
11080}
11081
11082
11083/**
11084 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11085 */
11086HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11087{
11088 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11089 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11090 {
11091 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11092 HMVMX_RETURN_UNEXPECTED_EXIT();
11093 }
11094
11095 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11096
11097 /*
11098 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11099 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11100 */
11101 uint32_t fIntrState = 0;
11102 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11103 AssertRCReturn(rc, rc);
11104
11105 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11106 if ( fBlockSti
11107 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11108 {
11109 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11110 }
11111
11112 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11113 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11114
11115 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11116 return VINF_SUCCESS;
11117}
11118
11119
11120/**
11121 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11122 */
11123HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11126 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11127}
11128
11129
11130/**
11131 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11132 */
11133HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11134{
11135 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11136 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11137}
11138
11139
11140/**
11141 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11142 */
11143HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11144{
11145 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11146 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11147
11148 /*
11149 * Get the state we need and update the exit history entry.
11150 */
11151 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11152 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11153 | CPUMCTX_EXTRN_CS);
11154 AssertRCReturn(rc, rc);
11155
11156 VBOXSTRICTRC rcStrict;
11157 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11158 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11159 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11160 if (!pExitRec)
11161 {
11162 /*
11163 * Regular CPUID instruction execution.
11164 */
11165 PVM pVM = pVCpu->CTX_SUFF(pVM);
11166 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11167 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11168 {
11169 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11170 Assert(pVmxTransient->cbInstr == 2);
11171 }
11172 else
11173 {
11174 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11175 rcStrict = VERR_EM_INTERPRETER;
11176 }
11177 }
11178 else
11179 {
11180 /*
11181 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11182 */
11183 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11184 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11185 AssertRCReturn(rc2, rc2);
11186
11187 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11188 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11189
11190 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11191 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11192
11193 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11194 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11195 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11196 }
11197 return VBOXSTRICTRC_TODO(rcStrict);
11198}
11199
11200
11201/**
11202 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11203 */
11204HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11205{
11206 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11207 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11208 AssertRCReturn(rc, rc);
11209
11210 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11211 return VINF_EM_RAW_EMULATE_INSTR;
11212
11213 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11214 HMVMX_RETURN_UNEXPECTED_EXIT();
11215}
11216
11217
11218/**
11219 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11220 */
11221HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11222{
11223 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11224 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11225 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11226 AssertRCReturn(rc, rc);
11227
11228 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11229 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11230 {
11231 /* If we get a spurious VM-exit when offsetting is enabled,
11232 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11233 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11234 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11235 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11236 | HM_CHANGED_GUEST_RFLAGS);
11237 }
11238 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11239 {
11240 rcStrict = VINF_SUCCESS;
11241 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11242 }
11243 return rcStrict;
11244}
11245
11246
11247/**
11248 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11249 */
11250HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11251{
11252 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11253 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11254 | CPUMCTX_EXTRN_TSC_AUX);
11255 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11256 AssertRCReturn(rc, rc);
11257
11258 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11259 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11260 {
11261 /* If we get a spurious VM-exit when offsetting is enabled,
11262 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11263 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11264 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11265 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11266 | HM_CHANGED_GUEST_RFLAGS);
11267 }
11268 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11269 {
11270 rcStrict = VINF_SUCCESS;
11271 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11272 }
11273 return rcStrict;
11274}
11275
11276
11277/**
11278 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11279 */
11280HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11281{
11282 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11283 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11284 | CPUMCTX_EXTRN_CR0
11285 | CPUMCTX_EXTRN_RFLAGS
11286 | CPUMCTX_EXTRN_SS);
11287 AssertRCReturn(rc, rc);
11288
11289 PVM pVM = pVCpu->CTX_SUFF(pVM);
11290 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11291 if (RT_LIKELY(rc == VINF_SUCCESS))
11292 {
11293 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11294 Assert(pVmxTransient->cbInstr == 2);
11295 }
11296 else
11297 {
11298 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11299 rc = VERR_EM_INTERPRETER;
11300 }
11301 return rc;
11302}
11303
11304
11305/**
11306 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11307 */
11308HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11309{
11310 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11311
11312 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11313 if (EMAreHypercallInstructionsEnabled(pVCpu))
11314 {
11315 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11316 | CPUMCTX_EXTRN_RFLAGS
11317 | CPUMCTX_EXTRN_CR0
11318 | CPUMCTX_EXTRN_SS
11319 | CPUMCTX_EXTRN_CS
11320 | CPUMCTX_EXTRN_EFER);
11321 AssertRCReturn(rc, rc);
11322
11323 /* Perform the hypercall. */
11324 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11325 if (rcStrict == VINF_SUCCESS)
11326 {
11327 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11328 AssertRCReturn(rc, rc);
11329 }
11330 else
11331 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11332 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11333 || RT_FAILURE(rcStrict));
11334
11335 /* If the hypercall changes anything other than guest's general-purpose registers,
11336 we would need to reload the guest changed bits here before VM-entry. */
11337 }
11338 else
11339 Log4Func(("Hypercalls not enabled\n"));
11340
11341 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11342 if (RT_FAILURE(rcStrict))
11343 {
11344 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11345 rcStrict = VINF_SUCCESS;
11346 }
11347
11348 return rcStrict;
11349}
11350
11351
11352/**
11353 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11354 */
11355HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11356{
11357 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11358 PVM pVM = pVCpu->CTX_SUFF(pVM);
11359 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11360
11361 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11362 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11363 AssertRCReturn(rc, rc);
11364
11365 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11366 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11367 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11368 else
11369 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11370 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11371 return rcStrict;
11372}
11373
11374
11375/**
11376 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11377 */
11378HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11379{
11380 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11381 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11382 | CPUMCTX_EXTRN_RFLAGS
11383 | CPUMCTX_EXTRN_SS);
11384 AssertRCReturn(rc, rc);
11385
11386 PVM pVM = pVCpu->CTX_SUFF(pVM);
11387 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11388 if (RT_LIKELY(rc == VINF_SUCCESS))
11389 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11390 else
11391 {
11392 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11393 rc = VERR_EM_INTERPRETER;
11394 }
11395 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11396 return rc;
11397}
11398
11399
11400/**
11401 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11402 */
11403HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11404{
11405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11406 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11407 | CPUMCTX_EXTRN_RFLAGS
11408 | CPUMCTX_EXTRN_SS);
11409 AssertRCReturn(rc, rc);
11410
11411 PVM pVM = pVCpu->CTX_SUFF(pVM);
11412 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11413 rc = VBOXSTRICTRC_VAL(rc2);
11414 if (RT_LIKELY( rc == VINF_SUCCESS
11415 || rc == VINF_EM_HALT))
11416 {
11417 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11418 AssertRCReturn(rc3, rc3);
11419
11420 if ( rc == VINF_EM_HALT
11421 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11422 rc = VINF_SUCCESS;
11423 }
11424 else
11425 {
11426 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11427 rc = VERR_EM_INTERPRETER;
11428 }
11429 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11430 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11431 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11432 return rc;
11433}
11434
11435
11436/**
11437 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11438 */
11439HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11440{
11441 /*
11442 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11443 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11444 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11445 * VMX root operation. If we get here, something funny is going on.
11446 *
11447 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11448 */
11449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11450 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11451 HMVMX_RETURN_UNEXPECTED_EXIT();
11452}
11453
11454
11455/**
11456 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11457 */
11458HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11459{
11460 /*
11461 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11462 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11463 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11464 * an SMI. If we get here, something funny is going on.
11465 *
11466 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11467 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11468 */
11469 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11470 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11471 HMVMX_RETURN_UNEXPECTED_EXIT();
11472}
11473
11474
11475/**
11476 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11477 */
11478HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11479{
11480 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11481 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11482 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11483 HMVMX_RETURN_UNEXPECTED_EXIT();
11484}
11485
11486
11487/**
11488 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11489 */
11490HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11491{
11492 /*
11493 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11494 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11495 * See Intel spec. 25.3 "Other Causes of VM-exits".
11496 */
11497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11498 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11499 HMVMX_RETURN_UNEXPECTED_EXIT();
11500}
11501
11502
11503/**
11504 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11505 * VM-exit.
11506 */
11507HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508{
11509 /*
11510 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11511 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11512 *
11513 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11514 * See Intel spec. "23.8 Restrictions on VMX operation".
11515 */
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 return VINF_SUCCESS;
11518}
11519
11520
11521/**
11522 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11523 * VM-exit.
11524 */
11525HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11526{
11527 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11528 return VINF_EM_RESET;
11529}
11530
11531
11532/**
11533 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11534 */
11535HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11536{
11537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11538 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11539
11540 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11541 AssertRCReturn(rc, rc);
11542
11543 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11544 rc = VINF_SUCCESS;
11545 else
11546 rc = VINF_EM_HALT;
11547
11548 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11549 if (rc != VINF_SUCCESS)
11550 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11551 return rc;
11552}
11553
11554
11555/**
11556 * VM-exit handler for instructions that result in a \#UD exception delivered to
11557 * the guest.
11558 */
11559HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11560{
11561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11562 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11563 return VINF_SUCCESS;
11564}
11565
11566
11567/**
11568 * VM-exit handler for expiry of the VMX preemption timer.
11569 */
11570HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11571{
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11573
11574 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11575 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11576
11577 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11578 PVM pVM = pVCpu->CTX_SUFF(pVM);
11579 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11580 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11581 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11582}
11583
11584
11585/**
11586 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11587 */
11588HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11589{
11590 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11591
11592 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11593 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11594 | CPUMCTX_EXTRN_CR4);
11595 AssertRCReturn(rc, rc);
11596
11597 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11598 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11599 : HM_CHANGED_XCPT_RAISED_MASK);
11600
11601 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11602
11603 return rcStrict;
11604}
11605
11606
11607/**
11608 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11609 */
11610HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11611{
11612 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11613 /** @todo Use VM-exit instruction information. */
11614 return VERR_EM_INTERPRETER;
11615}
11616
11617
11618/**
11619 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11620 * Error VM-exit.
11621 */
11622HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11623{
11624 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11625 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11626 AssertRCReturn(rc, rc);
11627
11628 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11629 NOREF(uInvalidReason);
11630
11631#ifdef VBOX_STRICT
11632 uint32_t fIntrState;
11633 RTHCUINTREG uHCReg;
11634 uint64_t u64Val;
11635 uint32_t u32Val;
11636
11637 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11638 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11639 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11640 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11641 AssertRCReturn(rc, rc);
11642
11643 Log4(("uInvalidReason %u\n", uInvalidReason));
11644 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11645 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11646 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11647 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11648
11649 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11650 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11651 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11652 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11653 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11654 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11655 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11656 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11657 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11658 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11659 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11660 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11661
11662 hmR0DumpRegs(pVCpu, pMixedCtx);
11663#else
11664 NOREF(pVmxTransient);
11665#endif
11666
11667 return VERR_VMX_INVALID_GUEST_STATE;
11668}
11669
11670
11671/**
11672 * VM-exit handler for VM-entry failure due to an MSR-load
11673 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11674 */
11675HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11676{
11677 NOREF(pVmxTransient);
11678 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11679 HMVMX_RETURN_UNEXPECTED_EXIT();
11680}
11681
11682
11683/**
11684 * VM-exit handler for VM-entry failure due to a machine-check event
11685 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11686 */
11687HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11688{
11689 NOREF(pVmxTransient);
11690 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11691 HMVMX_RETURN_UNEXPECTED_EXIT();
11692}
11693
11694
11695/**
11696 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11697 * theory.
11698 */
11699HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11700{
11701 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11702 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11703 return VERR_VMX_UNDEFINED_EXIT_CODE;
11704}
11705
11706
11707/**
11708 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11709 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11710 * Conditional VM-exit.
11711 */
11712HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11713{
11714 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11715
11716 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11717 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11718 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11719 return VERR_EM_INTERPRETER;
11720 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11721 HMVMX_RETURN_UNEXPECTED_EXIT();
11722}
11723
11724
11725/**
11726 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11727 */
11728HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11729{
11730 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11731
11732 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11733 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11734 return VERR_EM_INTERPRETER;
11735 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11736 HMVMX_RETURN_UNEXPECTED_EXIT();
11737}
11738
11739
11740/**
11741 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11742 */
11743HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11746
11747 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11748 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11749 | CPUMCTX_EXTRN_RFLAGS
11750 | CPUMCTX_EXTRN_SS);
11751 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11752 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11753 AssertRCReturn(rc, rc);
11754 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11755
11756#ifdef VBOX_STRICT
11757 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11758 {
11759 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11760 && pMixedCtx->ecx != MSR_K6_EFER)
11761 {
11762 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11763 pMixedCtx->ecx));
11764 HMVMX_RETURN_UNEXPECTED_EXIT();
11765 }
11766 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11767 {
11768 VMXMSREXITREAD enmRead;
11769 VMXMSREXITWRITE enmWrite;
11770 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11771 AssertRCReturn(rc2, rc2);
11772 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11773 {
11774 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11775 HMVMX_RETURN_UNEXPECTED_EXIT();
11776 }
11777 }
11778 }
11779#endif
11780
11781 PVM pVM = pVCpu->CTX_SUFF(pVM);
11782 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11783 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11784 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11786 if (RT_SUCCESS(rc))
11787 {
11788 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11789 Assert(pVmxTransient->cbInstr == 2);
11790 }
11791 return rc;
11792}
11793
11794
11795/**
11796 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11797 */
11798HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11799{
11800 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11801 PVM pVM = pVCpu->CTX_SUFF(pVM);
11802 int rc = VINF_SUCCESS;
11803
11804 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11805 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11806 | CPUMCTX_EXTRN_RFLAGS
11807 | CPUMCTX_EXTRN_SS);
11808 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11809 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11810 AssertRCReturn(rc, rc);
11811 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11812
11813 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11814 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11815 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11816
11817 if (RT_SUCCESS(rc))
11818 {
11819 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11820
11821 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11822 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11823 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11824 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11825 {
11826 /*
11827 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11828 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11829 * EMInterpretWrmsr() changes it.
11830 */
11831 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11832 }
11833 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11834 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11835 else if (pMixedCtx->ecx == MSR_K6_EFER)
11836 {
11837 /*
11838 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11839 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11840 * the other bits as well, SCE and NXE. See @bugref{7368}.
11841 */
11842 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11843 | HM_CHANGED_VMX_ENTRY_CTLS
11844 | HM_CHANGED_VMX_EXIT_CTLS);
11845 }
11846
11847 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11848 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11849 {
11850 switch (pMixedCtx->ecx)
11851 {
11852 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11853 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11854 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11855 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11856 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11857 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11858 default:
11859 {
11860 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11861 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11862 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11863 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11864 break;
11865 }
11866 }
11867 }
11868#ifdef VBOX_STRICT
11869 else
11870 {
11871 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11872 switch (pMixedCtx->ecx)
11873 {
11874 case MSR_IA32_SYSENTER_CS:
11875 case MSR_IA32_SYSENTER_EIP:
11876 case MSR_IA32_SYSENTER_ESP:
11877 case MSR_K8_FS_BASE:
11878 case MSR_K8_GS_BASE:
11879 {
11880 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11881 HMVMX_RETURN_UNEXPECTED_EXIT();
11882 }
11883
11884 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11885 default:
11886 {
11887 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11888 {
11889 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11890 if (pMixedCtx->ecx != MSR_K6_EFER)
11891 {
11892 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11893 pMixedCtx->ecx));
11894 HMVMX_RETURN_UNEXPECTED_EXIT();
11895 }
11896 }
11897
11898 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11899 {
11900 VMXMSREXITREAD enmRead;
11901 VMXMSREXITWRITE enmWrite;
11902 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11903 AssertRCReturn(rc2, rc2);
11904 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11905 {
11906 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11907 HMVMX_RETURN_UNEXPECTED_EXIT();
11908 }
11909 }
11910 break;
11911 }
11912 }
11913 }
11914#endif /* VBOX_STRICT */
11915 }
11916 return rc;
11917}
11918
11919
11920/**
11921 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11922 */
11923HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11924{
11925 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11926 /** @todo The guest has likely hit a contended spinlock. We might want to
11927 * poke a schedule different guest VCPU. */
11928 return VINF_EM_RAW_INTERRUPT;
11929}
11930
11931
11932/**
11933 * VM-exit handler for when the TPR value is lowered below the specified
11934 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11935 */
11936HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11937{
11938 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11939 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11940
11941 /*
11942 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11943 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11944 */
11945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11946 return VINF_SUCCESS;
11947}
11948
11949
11950/**
11951 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11952 * VM-exit.
11953 *
11954 * @retval VINF_SUCCESS when guest execution can continue.
11955 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11956 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11957 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11958 * interpreter.
11959 */
11960HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11961{
11962 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11963 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11964
11965 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11966 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11967 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11968 AssertRCReturn(rc, rc);
11969
11970 VBOXSTRICTRC rcStrict;
11971 PVM pVM = pVCpu->CTX_SUFF(pVM);
11972 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11973 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11974 switch (uAccessType)
11975 {
11976 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11977 {
11978 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11979 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11980 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11981 AssertMsg( rcStrict == VINF_SUCCESS
11982 || rcStrict == VINF_IEM_RAISED_XCPT
11983 || rcStrict == VINF_PGM_CHANGE_MODE
11984 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11985
11986 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11987 {
11988 case 0:
11989 {
11990 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11992 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11993 break;
11994 }
11995
11996 case 2:
11997 {
11998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11999 /* Nothing to do here, CR2 it's not part of the VMCS. */
12000 break;
12001 }
12002
12003 case 3:
12004 {
12005 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12006 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12007 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12008 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12009 break;
12010 }
12011
12012 case 4:
12013 {
12014 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12015 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12016 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12017 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12018 break;
12019 }
12020
12021 case 8:
12022 {
12023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12024 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12025 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12026 break;
12027 }
12028 default:
12029 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12030 break;
12031 }
12032 break;
12033 }
12034
12035 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12036 {
12037 Assert( !pVM->hm.s.fNestedPaging
12038 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12039 || pVCpu->hm.s.fUsingDebugLoop
12040 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12041 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12042 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12043 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12044
12045 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12046 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12047 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12048 AssertMsg( rcStrict == VINF_SUCCESS
12049 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12050#ifdef VBOX_WITH_STATISTICS
12051 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12052 {
12053 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12054 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12055 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12056 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12057 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12058 }
12059#endif
12060 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12061 VBOXSTRICTRC_VAL(rcStrict)));
12062 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12063 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12064 break;
12065 }
12066
12067 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12068 {
12069 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12070 AssertMsg( rcStrict == VINF_SUCCESS
12071 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12072
12073 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12075 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12076 break;
12077 }
12078
12079 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12080 {
12081 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12082 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12083 AssertMsg( rcStrict == VINF_SUCCESS
12084 || rcStrict == VINF_IEM_RAISED_XCPT
12085 || rcStrict == VINF_PGM_CHANGE_MODE,
12086 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12087
12088 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12089 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12090 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12091 break;
12092 }
12093
12094 default:
12095 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12096 VERR_VMX_UNEXPECTED_EXCEPTION);
12097 }
12098
12099 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12100 : HM_CHANGED_XCPT_RAISED_MASK);
12101 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12102 NOREF(pVM);
12103 return rcStrict;
12104}
12105
12106
12107/**
12108 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12109 * VM-exit.
12110 */
12111HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12112{
12113 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12114 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12115 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12116
12117 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12118 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12119 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12120 | CPUMCTX_EXTRN_SREG_MASK
12121 | CPUMCTX_EXTRN_EFER);
12122 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12123 AssertRCReturn(rc, rc);
12124
12125 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12126 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12127 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12128 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12129 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12130 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12131 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12132 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12133 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12134
12135 /*
12136 * Update exit history to see if this exit can be optimized.
12137 */
12138 VBOXSTRICTRC rcStrict;
12139 PCEMEXITREC pExitRec = NULL;
12140 if ( !fGstStepping
12141 && !fDbgStepping)
12142 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12143 !fIOString
12144 ? !fIOWrite
12145 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12146 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12147 : !fIOWrite
12148 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12149 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12150 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12151 if (!pExitRec)
12152 {
12153 /* I/O operation lookup arrays. */
12154 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12155 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12156 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12157 uint32_t const cbInstr = pVmxTransient->cbInstr;
12158 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12159 PVM pVM = pVCpu->CTX_SUFF(pVM);
12160 if (fIOString)
12161 {
12162 /*
12163 * INS/OUTS - I/O String instruction.
12164 *
12165 * Use instruction-information if available, otherwise fall back on
12166 * interpreting the instruction.
12167 */
12168 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12169 fIOWrite ? 'w' : 'r'));
12170 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12171 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12172 {
12173 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12174 AssertRCReturn(rc2, rc2);
12175 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12176 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12177 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12178 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12179 if (fIOWrite)
12180 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12181 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12182 else
12183 {
12184 /*
12185 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12186 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12187 * See Intel Instruction spec. for "INS".
12188 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12189 */
12190 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12191 }
12192 }
12193 else
12194 rcStrict = IEMExecOne(pVCpu);
12195
12196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12197 fUpdateRipAlready = true;
12198 }
12199 else
12200 {
12201 /*
12202 * IN/OUT - I/O instruction.
12203 */
12204 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12205 fIOWrite ? 'w' : 'r'));
12206 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12207 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12208 if (fIOWrite)
12209 {
12210 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12211 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12212 }
12213 else
12214 {
12215 uint32_t u32Result = 0;
12216 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12217 if (IOM_SUCCESS(rcStrict))
12218 {
12219 /* Save result of I/O IN instr. in AL/AX/EAX. */
12220 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12221 }
12222 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12223 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12224 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12225 }
12226 }
12227
12228 if (IOM_SUCCESS(rcStrict))
12229 {
12230 if (!fUpdateRipAlready)
12231 {
12232 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12233 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12234 }
12235
12236 /*
12237 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12238 * while booting Fedora 17 64-bit guest.
12239 *
12240 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12241 */
12242 if (fIOString)
12243 {
12244 /** @todo Single-step for INS/OUTS with REP prefix? */
12245 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12246 }
12247 else if ( !fDbgStepping
12248 && fGstStepping)
12249 {
12250 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12251 AssertRCReturn(rc, rc);
12252 }
12253
12254 /*
12255 * If any I/O breakpoints are armed, we need to check if one triggered
12256 * and take appropriate action.
12257 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12258 */
12259 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12260 AssertRCReturn(rc, rc);
12261
12262 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12263 * execution engines about whether hyper BPs and such are pending. */
12264 uint32_t const uDr7 = pMixedCtx->dr[7];
12265 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12266 && X86_DR7_ANY_RW_IO(uDr7)
12267 && (pMixedCtx->cr4 & X86_CR4_DE))
12268 || DBGFBpIsHwIoArmed(pVM)))
12269 {
12270 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12271
12272 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12273 VMMRZCallRing3Disable(pVCpu);
12274 HM_DISABLE_PREEMPT();
12275
12276 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12277
12278 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12279 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12280 {
12281 /* Raise #DB. */
12282 if (fIsGuestDbgActive)
12283 ASMSetDR6(pMixedCtx->dr[6]);
12284 if (pMixedCtx->dr[7] != uDr7)
12285 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12286
12287 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12288 }
12289 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12290 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12291 else if ( rcStrict2 != VINF_SUCCESS
12292 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12293 rcStrict = rcStrict2;
12294 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12295
12296 HM_RESTORE_PREEMPT();
12297 VMMRZCallRing3Enable(pVCpu);
12298 }
12299 }
12300
12301#ifdef VBOX_STRICT
12302 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12303 Assert(!fIOWrite);
12304 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12305 Assert(fIOWrite);
12306 else
12307 {
12308# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12309 * statuses, that the VMM device and some others may return. See
12310 * IOM_SUCCESS() for guidance. */
12311 AssertMsg( RT_FAILURE(rcStrict)
12312 || rcStrict == VINF_SUCCESS
12313 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12314 || rcStrict == VINF_EM_DBG_BREAKPOINT
12315 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12316 || rcStrict == VINF_EM_RAW_TO_R3
12317 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12318# endif
12319 }
12320#endif
12321 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12322 }
12323 else
12324 {
12325 /*
12326 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12327 */
12328 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12329 AssertRCReturn(rc2, rc2);
12330 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12331 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12332 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12333 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12334 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12335 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12336
12337 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12338 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12339
12340 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12341 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12342 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12343 }
12344 return rcStrict;
12345}
12346
12347
12348/**
12349 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12350 * VM-exit.
12351 */
12352HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12353{
12354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12355
12356 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12357 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12358 AssertRCReturn(rc, rc);
12359 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12360 {
12361 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12362 AssertRCReturn(rc, rc);
12363 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12364 {
12365 uint32_t uErrCode;
12366 RTGCUINTPTR GCPtrFaultAddress;
12367 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12368 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12369 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12370 if (fErrorCodeValid)
12371 {
12372 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12373 AssertRCReturn(rc, rc);
12374 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12375 }
12376 else
12377 uErrCode = 0;
12378
12379 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12380 && uVector == X86_XCPT_PF)
12381 GCPtrFaultAddress = pMixedCtx->cr2;
12382 else
12383 GCPtrFaultAddress = 0;
12384
12385 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12386 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12387
12388 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12389 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12390 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12391 }
12392 }
12393
12394 /* Fall back to the interpreter to emulate the task-switch. */
12395 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12396 return VERR_EM_INTERPRETER;
12397}
12398
12399
12400/**
12401 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12402 */
12403HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12404{
12405 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12406 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12407 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12408 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12409 AssertRCReturn(rc, rc);
12410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12411 return VINF_EM_DBG_STEPPED;
12412}
12413
12414
12415/**
12416 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12417 */
12418HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12419{
12420 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12421
12422 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12423
12424 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12425 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12426 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12427 {
12428 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12429 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12430 {
12431 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12432 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12433 }
12434 }
12435 else
12436 {
12437 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12438 rcStrict1 = VINF_SUCCESS;
12439 return rcStrict1;
12440 }
12441
12442 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12443 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12444 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12445 AssertRCReturn(rc, rc);
12446
12447 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12448 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12449 VBOXSTRICTRC rcStrict2;
12450 switch (uAccessType)
12451 {
12452 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12453 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12454 {
12455 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12456 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12457 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12458
12459 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12460 GCPhys &= PAGE_BASE_GC_MASK;
12461 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12462 PVM pVM = pVCpu->CTX_SUFF(pVM);
12463 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12464 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12465
12466 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12467 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12468 CPUMCTX2CORE(pMixedCtx), GCPhys);
12469 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12470 if ( rcStrict2 == VINF_SUCCESS
12471 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12472 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12473 {
12474 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12475 | HM_CHANGED_GUEST_RSP
12476 | HM_CHANGED_GUEST_RFLAGS
12477 | HM_CHANGED_GUEST_APIC_TPR);
12478 rcStrict2 = VINF_SUCCESS;
12479 }
12480 break;
12481 }
12482
12483 default:
12484 Log4Func(("uAccessType=%#x\n", uAccessType));
12485 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12486 break;
12487 }
12488
12489 if (rcStrict2 != VINF_SUCCESS)
12490 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12491 return rcStrict2;
12492}
12493
12494
12495/**
12496 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12497 * VM-exit.
12498 */
12499HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12500{
12501 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12502
12503 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12504 if (pVmxTransient->fWasGuestDebugStateActive)
12505 {
12506 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12507 HMVMX_RETURN_UNEXPECTED_EXIT();
12508 }
12509
12510 if ( !pVCpu->hm.s.fSingleInstruction
12511 && !pVmxTransient->fWasHyperDebugStateActive)
12512 {
12513 Assert(!DBGFIsStepping(pVCpu));
12514 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12515
12516 /* Don't intercept MOV DRx any more. */
12517 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12518 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12519 AssertRCReturn(rc, rc);
12520
12521 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12522 VMMRZCallRing3Disable(pVCpu);
12523 HM_DISABLE_PREEMPT();
12524
12525 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12526 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12527 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12528
12529 HM_RESTORE_PREEMPT();
12530 VMMRZCallRing3Enable(pVCpu);
12531
12532#ifdef VBOX_WITH_STATISTICS
12533 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12534 AssertRCReturn(rc, rc);
12535 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12536 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12537 else
12538 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12539#endif
12540 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12541 return VINF_SUCCESS;
12542 }
12543
12544 /*
12545 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12546 * Update the segment registers and DR7 from the CPU.
12547 */
12548 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12549 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12550 | CPUMCTX_EXTRN_DR7);
12551 AssertRCReturn(rc, rc);
12552 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12553
12554 PVM pVM = pVCpu->CTX_SUFF(pVM);
12555 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12556 {
12557 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12558 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12559 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12560 if (RT_SUCCESS(rc))
12561 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12563 }
12564 else
12565 {
12566 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12567 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12568 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12569 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12570 }
12571
12572 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12573 if (RT_SUCCESS(rc))
12574 {
12575 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12576 AssertRCReturn(rc2, rc2);
12577 return VINF_SUCCESS;
12578 }
12579 return rc;
12580}
12581
12582
12583/**
12584 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12585 * Conditional VM-exit.
12586 */
12587HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12588{
12589 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12590 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12591
12592 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12593 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12594 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12595 {
12596 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12597 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12598 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12599 {
12600 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12601 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12602 }
12603 }
12604 else
12605 {
12606 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12607 rcStrict1 = VINF_SUCCESS;
12608 return rcStrict1;
12609 }
12610
12611 /*
12612 * Get sufficent state and update the exit history entry.
12613 */
12614 RTGCPHYS GCPhys;
12615 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12616 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12617 AssertRCReturn(rc, rc);
12618
12619 VBOXSTRICTRC rcStrict;
12620 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12621 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12622 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12623 if (!pExitRec)
12624 {
12625 /*
12626 * If we succeed, resume guest execution.
12627 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12628 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12629 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12630 * weird case. See @bugref{6043}.
12631 */
12632 PVM pVM = pVCpu->CTX_SUFF(pVM);
12633 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12634 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12635 if ( rcStrict == VINF_SUCCESS
12636 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12637 || rcStrict == VERR_PAGE_NOT_PRESENT)
12638 {
12639 /* Successfully handled MMIO operation. */
12640 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12641 | HM_CHANGED_GUEST_RSP
12642 | HM_CHANGED_GUEST_RFLAGS
12643 | HM_CHANGED_GUEST_APIC_TPR);
12644 rcStrict = VINF_SUCCESS;
12645 }
12646 }
12647 else
12648 {
12649 /*
12650 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12651 */
12652 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12653 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12654 AssertRCReturn(rc2, rc2);
12655
12656 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12657 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12658
12659 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12660 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12661
12662 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12663 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12664 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12665 }
12666 return VBOXSTRICTRC_TODO(rcStrict);
12667}
12668
12669
12670/**
12671 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12672 * VM-exit.
12673 */
12674HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12675{
12676 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12677 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12678
12679 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12680 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12681 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12682 {
12683 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12684 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12685 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12686 }
12687 else
12688 {
12689 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12690 rcStrict1 = VINF_SUCCESS;
12691 return rcStrict1;
12692 }
12693
12694 RTGCPHYS GCPhys;
12695 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12696 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12697 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12698 AssertRCReturn(rc, rc);
12699
12700 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12701 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12702
12703 RTGCUINT uErrorCode = 0;
12704 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12705 uErrorCode |= X86_TRAP_PF_ID;
12706 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12707 uErrorCode |= X86_TRAP_PF_RW;
12708 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12709 uErrorCode |= X86_TRAP_PF_P;
12710
12711 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12712
12713 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12714 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12715
12716 /* Handle the pagefault trap for the nested shadow table. */
12717 PVM pVM = pVCpu->CTX_SUFF(pVM);
12718 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12719 TRPMResetTrap(pVCpu);
12720
12721 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12722 if ( rcStrict2 == VINF_SUCCESS
12723 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12724 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12725 {
12726 /* Successfully synced our nested page tables. */
12727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12728 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12729 | HM_CHANGED_GUEST_RSP
12730 | HM_CHANGED_GUEST_RFLAGS);
12731 return VINF_SUCCESS;
12732 }
12733
12734 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12735 return rcStrict2;
12736}
12737
12738/** @} */
12739
12740/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12741/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12742/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12743
12744/** @name VM-exit exception handlers.
12745 * @{
12746 */
12747
12748/**
12749 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12750 */
12751static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12752{
12753 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12754 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12755
12756 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12757 AssertRCReturn(rc, rc);
12758
12759 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12760 {
12761 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12762 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12763
12764 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12765 * provides VM-exit instruction length. If this causes problem later,
12766 * disassemble the instruction like it's done on AMD-V. */
12767 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12768 AssertRCReturn(rc2, rc2);
12769 return rc;
12770 }
12771
12772 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12773 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12774 return rc;
12775}
12776
12777
12778/**
12779 * VM-exit exception handler for \#BP (Breakpoint exception).
12780 */
12781static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12782{
12783 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12785
12786 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12787 AssertRCReturn(rc, rc);
12788
12789 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12790 if (rc == VINF_EM_RAW_GUEST_TRAP)
12791 {
12792 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12793 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12794 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12795 AssertRCReturn(rc, rc);
12796
12797 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12798 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12799 }
12800
12801 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12802 return rc;
12803}
12804
12805
12806/**
12807 * VM-exit exception handler for \#AC (alignment check exception).
12808 */
12809static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12810{
12811 RT_NOREF_PV(pMixedCtx);
12812 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12813
12814 /*
12815 * Re-inject it. We'll detect any nesting before getting here.
12816 */
12817 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12818 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12819 AssertRCReturn(rc, rc);
12820 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12821
12822 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12823 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12824 return VINF_SUCCESS;
12825}
12826
12827
12828/**
12829 * VM-exit exception handler for \#DB (Debug exception).
12830 */
12831static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12832{
12833 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12834 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12835
12836 /*
12837 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12838 * for processing.
12839 */
12840 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12841
12842 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12843 uint64_t uDR6 = X86_DR6_INIT_VAL;
12844 uDR6 |= ( pVmxTransient->uExitQualification
12845 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12846
12847 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12848 Log6Func(("rc=%Rrc\n", rc));
12849 if (rc == VINF_EM_RAW_GUEST_TRAP)
12850 {
12851 /*
12852 * The exception was for the guest. Update DR6, DR7.GD and
12853 * IA32_DEBUGCTL.LBR before forwarding it.
12854 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12855 */
12856 VMMRZCallRing3Disable(pVCpu);
12857 HM_DISABLE_PREEMPT();
12858
12859 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12860 pMixedCtx->dr[6] |= uDR6;
12861 if (CPUMIsGuestDebugStateActive(pVCpu))
12862 ASMSetDR6(pMixedCtx->dr[6]);
12863
12864 HM_RESTORE_PREEMPT();
12865 VMMRZCallRing3Enable(pVCpu);
12866
12867 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12868 AssertRCReturn(rc, rc);
12869
12870 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12871 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12872
12873 /* Paranoia. */
12874 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12875 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12876
12877 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12878 AssertRCReturn(rc, rc);
12879
12880 /*
12881 * Raise #DB in the guest.
12882 *
12883 * It is important to reflect exactly what the VM-exit gave us (preserving the
12884 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12885 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12886 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12887 *
12888 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12889 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12890 */
12891 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12892 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12893 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12894 AssertRCReturn(rc, rc);
12895 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12896 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12897 return VINF_SUCCESS;
12898 }
12899
12900 /*
12901 * Not a guest trap, must be a hypervisor related debug event then.
12902 * Update DR6 in case someone is interested in it.
12903 */
12904 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12905 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12906 CPUMSetHyperDR6(pVCpu, uDR6);
12907
12908 return rc;
12909}
12910
12911/**
12912 * VM-exit exception handler for \#GP (General-protection exception).
12913 *
12914 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12915 */
12916static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12917{
12918 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12919 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12920
12921 int rc;
12922 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12923 { /* likely */ }
12924 else
12925 {
12926#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12927 Assert(pVCpu->hm.s.fUsingDebugLoop);
12928#endif
12929 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12930 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12931 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12932 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12933 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12934 AssertRCReturn(rc, rc);
12935 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12936 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12937 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12938 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12939 return rc;
12940 }
12941
12942 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12943 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12944
12945 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12946 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12947 AssertRCReturn(rc, rc);
12948
12949 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12950 uint32_t cbOp = 0;
12951 PVM pVM = pVCpu->CTX_SUFF(pVM);
12952 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12953 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12954 if (RT_SUCCESS(rc))
12955 {
12956 rc = VINF_SUCCESS;
12957 Assert(cbOp == pDis->cbInstr);
12958 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12959 switch (pDis->pCurInstr->uOpcode)
12960 {
12961 case OP_CLI:
12962 {
12963 pMixedCtx->eflags.Bits.u1IF = 0;
12964 pMixedCtx->eflags.Bits.u1RF = 0;
12965 pMixedCtx->rip += pDis->cbInstr;
12966 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12967 if ( !fDbgStepping
12968 && pMixedCtx->eflags.Bits.u1TF)
12969 {
12970 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12971 AssertRCReturn(rc, rc);
12972 }
12973 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12974 break;
12975 }
12976
12977 case OP_STI:
12978 {
12979 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12980 pMixedCtx->eflags.Bits.u1IF = 1;
12981 pMixedCtx->eflags.Bits.u1RF = 0;
12982 pMixedCtx->rip += pDis->cbInstr;
12983 if (!fOldIF)
12984 {
12985 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12986 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12987 }
12988 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12989 if ( !fDbgStepping
12990 && pMixedCtx->eflags.Bits.u1TF)
12991 {
12992 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12993 AssertRCReturn(rc, rc);
12994 }
12995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12996 break;
12997 }
12998
12999 case OP_HLT:
13000 {
13001 rc = VINF_EM_HALT;
13002 pMixedCtx->rip += pDis->cbInstr;
13003 pMixedCtx->eflags.Bits.u1RF = 0;
13004 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13005 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13006 break;
13007 }
13008
13009 case OP_POPF:
13010 {
13011 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13012 uint32_t cbParm;
13013 uint32_t uMask;
13014 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13015 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13016 {
13017 cbParm = 4;
13018 uMask = 0xffffffff;
13019 }
13020 else
13021 {
13022 cbParm = 2;
13023 uMask = 0xffff;
13024 }
13025
13026 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13027 RTGCPTR GCPtrStack = 0;
13028 X86EFLAGS Eflags;
13029 Eflags.u32 = 0;
13030 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13031 &GCPtrStack);
13032 if (RT_SUCCESS(rc))
13033 {
13034 Assert(sizeof(Eflags.u32) >= cbParm);
13035 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13036 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13037 }
13038 if (RT_FAILURE(rc))
13039 {
13040 rc = VERR_EM_INTERPRETER;
13041 break;
13042 }
13043 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13044 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13045 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13046 pMixedCtx->esp += cbParm;
13047 pMixedCtx->esp &= uMask;
13048 pMixedCtx->rip += pDis->cbInstr;
13049 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13050 | HM_CHANGED_GUEST_RSP
13051 | HM_CHANGED_GUEST_RFLAGS);
13052 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13053 POPF restores EFLAGS.TF. */
13054 if ( !fDbgStepping
13055 && fGstStepping)
13056 {
13057 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13058 AssertRCReturn(rc, rc);
13059 }
13060 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13061 break;
13062 }
13063
13064 case OP_PUSHF:
13065 {
13066 uint32_t cbParm;
13067 uint32_t uMask;
13068 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13069 {
13070 cbParm = 4;
13071 uMask = 0xffffffff;
13072 }
13073 else
13074 {
13075 cbParm = 2;
13076 uMask = 0xffff;
13077 }
13078
13079 /* Get the stack pointer & push the contents of eflags onto the stack. */
13080 RTGCPTR GCPtrStack = 0;
13081 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13082 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13083 if (RT_FAILURE(rc))
13084 {
13085 rc = VERR_EM_INTERPRETER;
13086 break;
13087 }
13088 X86EFLAGS Eflags = pMixedCtx->eflags;
13089 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13090 Eflags.Bits.u1RF = 0;
13091 Eflags.Bits.u1VM = 0;
13092
13093 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13094 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13095 {
13096 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13097 rc = VERR_EM_INTERPRETER;
13098 break;
13099 }
13100 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13101 pMixedCtx->esp -= cbParm;
13102 pMixedCtx->esp &= uMask;
13103 pMixedCtx->rip += pDis->cbInstr;
13104 pMixedCtx->eflags.Bits.u1RF = 0;
13105 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13106 | HM_CHANGED_GUEST_RSP
13107 | HM_CHANGED_GUEST_RFLAGS);
13108 if ( !fDbgStepping
13109 && pMixedCtx->eflags.Bits.u1TF)
13110 {
13111 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13112 AssertRCReturn(rc, rc);
13113 }
13114 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13115 break;
13116 }
13117
13118 case OP_IRET:
13119 {
13120 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13121 * instruction reference. */
13122 RTGCPTR GCPtrStack = 0;
13123 uint32_t uMask = 0xffff;
13124 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13125 uint16_t aIretFrame[3];
13126 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13127 {
13128 rc = VERR_EM_INTERPRETER;
13129 break;
13130 }
13131 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13132 &GCPtrStack);
13133 if (RT_SUCCESS(rc))
13134 {
13135 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13136 PGMACCESSORIGIN_HM));
13137 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13138 }
13139 if (RT_FAILURE(rc))
13140 {
13141 rc = VERR_EM_INTERPRETER;
13142 break;
13143 }
13144 pMixedCtx->eip = 0;
13145 pMixedCtx->ip = aIretFrame[0];
13146 pMixedCtx->cs.Sel = aIretFrame[1];
13147 pMixedCtx->cs.ValidSel = aIretFrame[1];
13148 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13149 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13150 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13151 pMixedCtx->sp += sizeof(aIretFrame);
13152 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13153 | HM_CHANGED_GUEST_CS
13154 | HM_CHANGED_GUEST_RSP
13155 | HM_CHANGED_GUEST_RFLAGS);
13156 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13157 if ( !fDbgStepping
13158 && fGstStepping)
13159 {
13160 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13161 AssertRCReturn(rc, rc);
13162 }
13163 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13164 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13165 break;
13166 }
13167
13168 case OP_INT:
13169 {
13170 uint16_t uVector = pDis->Param1.uValue & 0xff;
13171 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13172 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13173 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13174 break;
13175 }
13176
13177 case OP_INTO:
13178 {
13179 if (pMixedCtx->eflags.Bits.u1OF)
13180 {
13181 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13182 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13183 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13184 }
13185 else
13186 {
13187 pMixedCtx->eflags.Bits.u1RF = 0;
13188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13189 }
13190 break;
13191 }
13192
13193 default:
13194 {
13195 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13196 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13197 EMCODETYPE_SUPERVISOR);
13198 rc = VBOXSTRICTRC_VAL(rc2);
13199 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13200 /** @todo We have to set pending-debug exceptions here when the guest is
13201 * single-stepping depending on the instruction that was interpreted. */
13202 Log4Func(("#GP rc=%Rrc\n", rc));
13203 break;
13204 }
13205 }
13206 }
13207 else
13208 rc = VERR_EM_INTERPRETER;
13209
13210 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13211 ("#GP Unexpected rc=%Rrc\n", rc));
13212 return rc;
13213}
13214
13215
13216/**
13217 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13218 * the exception reported in the VMX transient structure back into the VM.
13219 *
13220 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13221 * up-to-date.
13222 */
13223static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13224{
13225 RT_NOREF_PV(pMixedCtx);
13226 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13227#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13228 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13229 ("uVector=%#x u32XcptBitmap=%#X32\n",
13230 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13231#endif
13232
13233 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13234 hmR0VmxCheckExitDueToEventDelivery(). */
13235 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13236 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13237 AssertRCReturn(rc, rc);
13238 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13239
13240#ifdef DEBUG_ramshankar
13241 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13242 | CPUMCTX_EXTRN_RIP);
13243 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13244 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13245#endif
13246
13247 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13248 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13249 return VINF_SUCCESS;
13250}
13251
13252
13253/**
13254 * VM-exit exception handler for \#PF (Page-fault exception).
13255 */
13256static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13257{
13258 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13259 PVM pVM = pVCpu->CTX_SUFF(pVM);
13260 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13261 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13262 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13263 AssertRCReturn(rc, rc);
13264
13265 if (!pVM->hm.s.fNestedPaging)
13266 { /* likely */ }
13267 else
13268 {
13269#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13270 Assert(pVCpu->hm.s.fUsingDebugLoop);
13271#endif
13272 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13273 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13274 {
13275 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13276 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13277 }
13278 else
13279 {
13280 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13281 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13282 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13283 }
13284 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13285 return rc;
13286 }
13287
13288 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13289 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13290 if (pVmxTransient->fVectoringPF)
13291 {
13292 Assert(pVCpu->hm.s.Event.fPending);
13293 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13294 }
13295
13296 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13297 AssertRCReturn(rc, rc);
13298
13299 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13300 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13301
13302 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13303 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13304 (RTGCPTR)pVmxTransient->uExitQualification);
13305
13306 Log4Func(("#PF: rc=%Rrc\n", rc));
13307 if (rc == VINF_SUCCESS)
13308 {
13309 /*
13310 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13311 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13312 */
13313 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13314 TRPMResetTrap(pVCpu);
13315 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13316 return rc;
13317 }
13318
13319 if (rc == VINF_EM_RAW_GUEST_TRAP)
13320 {
13321 if (!pVmxTransient->fVectoringDoublePF)
13322 {
13323 /* It's a guest page fault and needs to be reflected to the guest. */
13324 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13325 TRPMResetTrap(pVCpu);
13326 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13327 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13328 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13329 }
13330 else
13331 {
13332 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13333 TRPMResetTrap(pVCpu);
13334 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13335 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13336 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13337 }
13338
13339 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13340 return VINF_SUCCESS;
13341 }
13342
13343 TRPMResetTrap(pVCpu);
13344 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13345 return rc;
13346}
13347
13348/** @} */
13349
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette