VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72820

Last change on this file since 72820 was 72820, checked in by vboxsync, 6 years ago

VMM/HMVMXR0,HMSVMR0: Compiler barriers with volatiles for fCtxChanged.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 573.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72820 2018-07-03 10:16:28Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
573 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
574 * VERR_VMX_INVALID_VMCS_FIELD.
575 * @param rc The error code.
576 */
577static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
578{
579 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
580 || rc == VERR_VMX_UNABLE_TO_START_VM)
581 {
582 AssertPtrReturnVoid(pVCpu);
583 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
584 }
585 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
586}
587
588
589/**
590 * Reads the VM-entry interruption-information field from the VMCS into the VMX
591 * transient structure.
592 *
593 * @returns VBox status code.
594 * @param pVmxTransient Pointer to the VMX transient structure.
595 *
596 * @remarks No-long-jump zone!!!
597 */
598DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
599{
600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
601 AssertRCReturn(rc, rc);
602 return VINF_SUCCESS;
603}
604
605#ifdef VBOX_STRICT
606/**
607 * Reads the VM-entry exception error code field from the VMCS into
608 * the VMX transient structure.
609 *
610 * @returns VBox status code.
611 * @param pVmxTransient Pointer to the VMX transient structure.
612 *
613 * @remarks No-long-jump zone!!!
614 */
615DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
616{
617 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
618 AssertRCReturn(rc, rc);
619 return VINF_SUCCESS;
620}
621
622
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641/**
642 * Reads the VM-exit interruption-information field from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
653 AssertRCReturn(rc,rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit interruption error code from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction length field from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction-information field from the VMCS into
700 * the VMX transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the exit code qualification from the VMCS into the VMX transient
719 * structure.
720 *
721 * @returns VBox status code.
722 * @param pVCpu The cross context virtual CPU structure of the
723 * calling EMT. (Required for the VMCS cache case.)
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
729 {
730 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Reads the IDT-vectoring information field from the VMCS into the VMX
740 * transient structure.
741 *
742 * @returns VBox status code.
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 *
745 * @remarks No-long-jump zone!!!
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Reads the IDT-vectoring error code from the VMCS into the VMX
761 * transient structure.
762 *
763 * @returns VBox status code.
764 * @param pVmxTransient Pointer to the VMX transient structure.
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Enters VMX root mode operation on the current CPU.
780 *
781 * @returns VBox status code.
782 * @param pVM The cross context VM structure. Can be
783 * NULL, after a resume.
784 * @param HCPhysCpuPage Physical address of the VMXON region.
785 * @param pvCpuPage Pointer to the VMXON region.
786 */
787static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
788{
789 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
790 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
791 Assert(pvCpuPage);
792 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793
794 if (pVM)
795 {
796 /* Write the VMCS revision dword to the VMXON region. */
797 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
798 }
799
800 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
801 RTCCUINTREG fEFlags = ASMIntDisableFlags();
802
803 /* Enable the VMX bit in CR4 if necessary. */
804 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
805
806 /* Enter VMX root mode. */
807 int rc = VMXEnable(HCPhysCpuPage);
808 if (RT_FAILURE(rc))
809 {
810 if (!(uOldCr4 & X86_CR4_VMXE))
811 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
812
813 if (pVM)
814 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
815 }
816
817 /* Restore interrupts. */
818 ASMSetFlags(fEFlags);
819 return rc;
820}
821
822
823/**
824 * Exits VMX root mode operation on the current CPU.
825 *
826 * @returns VBox status code.
827 */
828static int hmR0VmxLeaveRootMode(void)
829{
830 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
831
832 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
833 RTCCUINTREG fEFlags = ASMIntDisableFlags();
834
835 /* If we're for some reason not in VMX root mode, then don't leave it. */
836 RTCCUINTREG uHostCR4 = ASMGetCR4();
837
838 int rc;
839 if (uHostCR4 & X86_CR4_VMXE)
840 {
841 /* Exit VMX root mode and clear the VMX bit in CR4. */
842 VMXDisable();
843 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
844 rc = VINF_SUCCESS;
845 }
846 else
847 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
848
849 /* Restore interrupts. */
850 ASMSetFlags(fEFlags);
851 return rc;
852}
853
854
855/**
856 * Allocates and maps one physically contiguous page. The allocated page is
857 * zero'd out. (Used by various VT-x structures).
858 *
859 * @returns IPRT status code.
860 * @param pMemObj Pointer to the ring-0 memory object.
861 * @param ppVirt Where to store the virtual address of the
862 * allocation.
863 * @param pHCPhys Where to store the physical address of the
864 * allocation.
865 */
866static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
867{
868 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
869 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
871
872 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
873 if (RT_FAILURE(rc))
874 return rc;
875 *ppVirt = RTR0MemObjAddress(*pMemObj);
876 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
877 ASMMemZero32(*ppVirt, PAGE_SIZE);
878 return VINF_SUCCESS;
879}
880
881
882/**
883 * Frees and unmaps an allocated physical page.
884 *
885 * @param pMemObj Pointer to the ring-0 memory object.
886 * @param ppVirt Where to re-initialize the virtual address of
887 * allocation as 0.
888 * @param pHCPhys Where to re-initialize the physical address of the
889 * allocation as 0.
890 */
891static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
892{
893 AssertPtr(pMemObj);
894 AssertPtr(ppVirt);
895 AssertPtr(pHCPhys);
896 if (*pMemObj != NIL_RTR0MEMOBJ)
897 {
898 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
899 AssertRC(rc);
900 *pMemObj = NIL_RTR0MEMOBJ;
901 *ppVirt = 0;
902 *pHCPhys = 0;
903 }
904}
905
906
907/**
908 * Worker function to free VT-x related structures.
909 *
910 * @returns IPRT status code.
911 * @param pVM The cross context VM structure.
912 */
913static void hmR0VmxStructsFree(PVM pVM)
914{
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 AssertPtr(pVCpu);
919
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
922
923 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
925
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
927 }
928
929 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
930#ifdef VBOX_WITH_CRASHDUMP_MAGIC
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
932#endif
933}
934
935
936/**
937 * Worker function to allocate VT-x related VM structures.
938 *
939 * @returns IPRT status code.
940 * @param pVM The cross context VM structure.
941 */
942static int hmR0VmxStructsAlloc(PVM pVM)
943{
944 /*
945 * Initialize members up-front so we can cleanup properly on allocation failure.
946 */
947#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVM->hm.s.vmx.HCPhys##a_Name = 0;
951
952#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
953 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
954 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
955 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
956
957#ifdef VBOX_WITH_CRASHDUMP_MAGIC
958 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
959#endif
960 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
961
962 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
963 for (VMCPUID i = 0; i < pVM->cCpus; i++)
964 {
965 PVMCPU pVCpu = &pVM->aCpus[i];
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
970 }
971#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
972#undef VMXLOCAL_INIT_VM_MEMOBJ
973
974 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
975 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
976 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
977 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
978
979 /*
980 * Allocate all the VT-x structures.
981 */
982 int rc = VINF_SUCCESS;
983#ifdef VBOX_WITH_CRASHDUMP_MAGIC
984 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
988 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
989#endif
990
991 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
992 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
993 {
994 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
995 &pVM->hm.s.vmx.HCPhysApicAccess);
996 if (RT_FAILURE(rc))
997 goto cleanup;
998 }
999
1000 /*
1001 * Initialize per-VCPU VT-x structures.
1002 */
1003 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1004 {
1005 PVMCPU pVCpu = &pVM->aCpus[i];
1006 AssertPtr(pVCpu);
1007
1008 /* Allocate the VM control structure (VMCS). */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012
1013 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1014 if ( PDMHasApic(pVM)
1015 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1016 {
1017 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1018 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1019 if (RT_FAILURE(rc))
1020 goto cleanup;
1021 }
1022
1023 /*
1024 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1025 * transparent accesses of specific MSRs.
1026 *
1027 * If the condition for enabling MSR bitmaps changes here, don't forget to
1028 * update HMAreMsrBitmapsAvailable().
1029 */
1030 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1031 {
1032 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1033 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1034 if (RT_FAILURE(rc))
1035 goto cleanup;
1036 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1037 }
1038
1039 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043
1044 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1045 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1046 if (RT_FAILURE(rc))
1047 goto cleanup;
1048 }
1049
1050 return VINF_SUCCESS;
1051
1052cleanup:
1053 hmR0VmxStructsFree(pVM);
1054 return rc;
1055}
1056
1057
1058/**
1059 * Does global VT-x initialization (called during module initialization).
1060 *
1061 * @returns VBox status code.
1062 */
1063VMMR0DECL(int) VMXR0GlobalInit(void)
1064{
1065#ifdef HMVMX_USE_FUNCTION_TABLE
1066 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1067# ifdef VBOX_STRICT
1068 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1069 Assert(g_apfnVMExitHandlers[i]);
1070# endif
1071#endif
1072 return VINF_SUCCESS;
1073}
1074
1075
1076/**
1077 * Does global VT-x termination (called during module termination).
1078 */
1079VMMR0DECL(void) VMXR0GlobalTerm()
1080{
1081 /* Nothing to do currently. */
1082}
1083
1084
1085/**
1086 * Sets up and activates VT-x on the current CPU.
1087 *
1088 * @returns VBox status code.
1089 * @param pHostCpu Pointer to the global CPU info struct.
1090 * @param pVM The cross context VM structure. Can be
1091 * NULL after a host resume operation.
1092 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1093 * fEnabledByHost is @c true).
1094 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1095 * @a fEnabledByHost is @c true).
1096 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1097 * enable VT-x on the host.
1098 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1099 */
1100VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1101 void *pvMsrs)
1102{
1103 Assert(pHostCpu);
1104 Assert(pvMsrs);
1105 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1106
1107 /* Enable VT-x if it's not already enabled by the host. */
1108 if (!fEnabledByHost)
1109 {
1110 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1111 if (RT_FAILURE(rc))
1112 return rc;
1113 }
1114
1115 /*
1116 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1117 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1118 * invalidated when flushing by VPID.
1119 */
1120 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1121 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1122 {
1123 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1124 pHostCpu->fFlushAsidBeforeUse = false;
1125 }
1126 else
1127 pHostCpu->fFlushAsidBeforeUse = true;
1128
1129 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1130 ++pHostCpu->cTlbFlushes;
1131
1132 return VINF_SUCCESS;
1133}
1134
1135
1136/**
1137 * Deactivates VT-x on the current CPU.
1138 *
1139 * @returns VBox status code.
1140 * @param pHostCpu Pointer to the global CPU info struct.
1141 * @param pvCpuPage Pointer to the VMXON region.
1142 * @param HCPhysCpuPage Physical address of the VMXON region.
1143 *
1144 * @remarks This function should never be called when SUPR0EnableVTx() or
1145 * similar was used to enable VT-x on the host.
1146 */
1147VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1148{
1149 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1150
1151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1152 return hmR0VmxLeaveRootMode();
1153}
1154
1155
1156/**
1157 * Sets the permission bits for the specified MSR in the MSR bitmap.
1158 *
1159 * @param pVCpu The cross context virtual CPU structure.
1160 * @param uMsr The MSR value.
1161 * @param enmRead Whether reading this MSR causes a VM-exit.
1162 * @param enmWrite Whether writing this MSR causes a VM-exit.
1163 */
1164static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1165{
1166 int32_t iBit;
1167 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1168
1169 /*
1170 * Layout:
1171 * 0x000 - 0x3ff - Low MSR read bits
1172 * 0x400 - 0x7ff - High MSR read bits
1173 * 0x800 - 0xbff - Low MSR write bits
1174 * 0xc00 - 0xfff - High MSR write bits
1175 */
1176 if (uMsr <= 0x00001FFF)
1177 iBit = uMsr;
1178 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1179 {
1180 iBit = uMsr - UINT32_C(0xC0000000);
1181 pbMsrBitmap += 0x400;
1182 }
1183 else
1184 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1185
1186 Assert(iBit <= 0x1fff);
1187 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1188 ASMBitSet(pbMsrBitmap, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap, iBit);
1191
1192 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1193 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1194 else
1195 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1196}
1197
1198
1199#ifdef VBOX_STRICT
1200/**
1201 * Gets the permission bits for the specified MSR in the MSR bitmap.
1202 *
1203 * @returns VBox status code.
1204 * @retval VINF_SUCCESS if the specified MSR is found.
1205 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1206 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1207 *
1208 * @param pVCpu The cross context virtual CPU structure.
1209 * @param uMsr The MSR.
1210 * @param penmRead Where to store the read permissions.
1211 * @param penmWrite Where to store the write permissions.
1212 */
1213static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1214{
1215 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1216 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1217 int32_t iBit;
1218 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1219
1220 /* See hmR0VmxSetMsrPermission() for the layout. */
1221 if (uMsr <= 0x00001FFF)
1222 iBit = uMsr;
1223 else if ( uMsr >= 0xC0000000
1224 && uMsr <= 0xC0001FFF)
1225 {
1226 iBit = (uMsr - 0xC0000000);
1227 pbMsrBitmap += 0x400;
1228 }
1229 else
1230 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1231
1232 Assert(iBit <= 0x1fff);
1233 if (ASMBitTest(pbMsrBitmap, iBit))
1234 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1235 else
1236 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1237
1238 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1239 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1240 else
1241 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1242 return VINF_SUCCESS;
1243}
1244#endif /* VBOX_STRICT */
1245
1246
1247/**
1248 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1249 * area.
1250 *
1251 * @returns VBox status code.
1252 * @param pVCpu The cross context virtual CPU structure.
1253 * @param cMsrs The number of MSRs.
1254 */
1255static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1256{
1257 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1258 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1259 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1260 {
1261 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1262 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1264 }
1265
1266 /* Update number of guest MSRs to load/store across the world-switch. */
1267 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1269
1270 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1271 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1272 AssertRCReturn(rc, rc);
1273
1274 /* Update the VCPU's copy of the MSR count. */
1275 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1276
1277 return VINF_SUCCESS;
1278}
1279
1280
1281/**
1282 * Adds a new (or updates the value of an existing) guest/host MSR
1283 * pair to be swapped during the world-switch as part of the
1284 * auto-load/store MSR area in the VMCS.
1285 *
1286 * @returns VBox status code.
1287 * @param pVCpu The cross context virtual CPU structure.
1288 * @param uMsr The MSR.
1289 * @param uGuestMsrValue Value of the guest MSR.
1290 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1291 * necessary.
1292 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1293 * its value was updated. Optional, can be NULL.
1294 */
1295static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1296 bool *pfAddedAndUpdated)
1297{
1298 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1299 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1300 uint32_t i;
1301 for (i = 0; i < cMsrs; i++)
1302 {
1303 if (pGuestMsr->u32Msr == uMsr)
1304 break;
1305 pGuestMsr++;
1306 }
1307
1308 bool fAdded = false;
1309 if (i == cMsrs)
1310 {
1311 ++cMsrs;
1312 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1313 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1314
1315 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1316 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1317 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1318
1319 fAdded = true;
1320 }
1321
1322 /* Update the MSR values in the auto-load/store MSR area. */
1323 pGuestMsr->u32Msr = uMsr;
1324 pGuestMsr->u64Value = uGuestMsrValue;
1325
1326 /* Create/update the MSR slot in the host MSR area. */
1327 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1328 pHostMsr += i;
1329 pHostMsr->u32Msr = uMsr;
1330
1331 /*
1332 * Update the host MSR only when requested by the caller AND when we're
1333 * adding it to the auto-load/store area. Otherwise, it would have been
1334 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1335 */
1336 bool fUpdatedMsrValue = false;
1337 if ( fAdded
1338 && fUpdateHostMsr)
1339 {
1340 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1341 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1342 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1343 fUpdatedMsrValue = true;
1344 }
1345
1346 if (pfAddedAndUpdated)
1347 *pfAddedAndUpdated = fUpdatedMsrValue;
1348 return VINF_SUCCESS;
1349}
1350
1351
1352/**
1353 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1354 * auto-load/store MSR area in the VMCS.
1355 *
1356 * @returns VBox status code.
1357 * @param pVCpu The cross context virtual CPU structure.
1358 * @param uMsr The MSR.
1359 */
1360static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1361{
1362 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1363 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1364 for (uint32_t i = 0; i < cMsrs; i++)
1365 {
1366 /* Find the MSR. */
1367 if (pGuestMsr->u32Msr == uMsr)
1368 {
1369 /* If it's the last MSR, simply reduce the count. */
1370 if (i == cMsrs - 1)
1371 {
1372 --cMsrs;
1373 break;
1374 }
1375
1376 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1377 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 pLastGuestMsr += cMsrs - 1;
1379 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1380 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1381
1382 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1383 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 pLastHostMsr += cMsrs - 1;
1385 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1386 pHostMsr->u64Value = pLastHostMsr->u64Value;
1387 --cMsrs;
1388 break;
1389 }
1390 pGuestMsr++;
1391 }
1392
1393 /* Update the VMCS if the count changed (meaning the MSR was found). */
1394 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1395 {
1396 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1397 AssertRCReturn(rc, rc);
1398
1399 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1400 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1401 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1402
1403 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1404 return VINF_SUCCESS;
1405 }
1406
1407 return VERR_NOT_FOUND;
1408}
1409
1410
1411/**
1412 * Checks if the specified guest MSR is part of the auto-load/store area in
1413 * the VMCS.
1414 *
1415 * @returns true if found, false otherwise.
1416 * @param pVCpu The cross context virtual CPU structure.
1417 * @param uMsr The MSR to find.
1418 */
1419static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1420{
1421 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1422 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1423
1424 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1425 {
1426 if (pGuestMsr->u32Msr == uMsr)
1427 return true;
1428 }
1429 return false;
1430}
1431
1432
1433/**
1434 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1435 *
1436 * @param pVCpu The cross context virtual CPU structure.
1437 *
1438 * @remarks No-long-jump zone!!!
1439 */
1440static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1441{
1442 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1443 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1444 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1445 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1446
1447 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1448 {
1449 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1450
1451 /*
1452 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1453 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1454 */
1455 if (pHostMsr->u32Msr == MSR_K6_EFER)
1456 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1457 else
1458 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1459 }
1460
1461 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1462}
1463
1464
1465/**
1466 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1467 * perform lazy restoration of the host MSRs while leaving VT-x.
1468 *
1469 * @param pVCpu The cross context virtual CPU structure.
1470 *
1471 * @remarks No-long-jump zone!!!
1472 */
1473static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1474{
1475 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1476
1477 /*
1478 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1479 */
1480 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1481 {
1482 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1483#if HC_ARCH_BITS == 64
1484 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1485 {
1486 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1487 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1488 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1489 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1490 }
1491#endif
1492 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1493 }
1494}
1495
1496
1497/**
1498 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1499 * lazily while leaving VT-x.
1500 *
1501 * @returns true if it does, false otherwise.
1502 * @param pVCpu The cross context virtual CPU structure.
1503 * @param uMsr The MSR to check.
1504 */
1505static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1506{
1507 NOREF(pVCpu);
1508#if HC_ARCH_BITS == 64
1509 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1510 {
1511 switch (uMsr)
1512 {
1513 case MSR_K8_LSTAR:
1514 case MSR_K6_STAR:
1515 case MSR_K8_SF_MASK:
1516 case MSR_K8_KERNEL_GS_BASE:
1517 return true;
1518 }
1519 }
1520#else
1521 RT_NOREF(pVCpu, uMsr);
1522#endif
1523 return false;
1524}
1525
1526
1527/**
1528 * Loads a set of guests MSRs to allow read/passthru to the guest.
1529 *
1530 * The name of this function is slightly confusing. This function does NOT
1531 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1532 * common prefix for functions dealing with "lazy restoration" of the shared
1533 * MSRs.
1534 *
1535 * @param pVCpu The cross context virtual CPU structure.
1536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1537 * out-of-sync. Make sure to update the required fields
1538 * before using them.
1539 *
1540 * @remarks No-long-jump zone!!!
1541 */
1542static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1543{
1544 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1545 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1546
1547 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1548#if HC_ARCH_BITS == 64
1549 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1550 {
1551 /*
1552 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1553 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1554 * we can skip a few MSR writes.
1555 *
1556 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1557 * guest MSR values in the guest-CPU context might be different to what's currently
1558 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1559 * CPU, see @bugref{8728}.
1560 */
1561 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1562 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1563 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1564 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1565 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1566 {
1567#ifdef VBOX_STRICT
1568 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1569 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1570 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1571 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1572#endif
1573 }
1574 else
1575 {
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1577 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1578 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1579 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1580 }
1581 }
1582#else
1583 RT_NOREF(pMixedCtx);
1584#endif
1585 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1586}
1587
1588
1589/**
1590 * Performs lazy restoration of the set of host MSRs if they were previously
1591 * loaded with guest MSR values.
1592 *
1593 * @param pVCpu The cross context virtual CPU structure.
1594 *
1595 * @remarks No-long-jump zone!!!
1596 * @remarks The guest MSRs should have been saved back into the guest-CPU
1597 * context by hmR0VmxImportGuestState()!!!
1598 */
1599static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1600{
1601 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1603
1604 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1605 {
1606 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1607#if HC_ARCH_BITS == 64
1608 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1609 {
1610 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1611 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1612 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1613 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1614 }
1615#endif
1616 }
1617 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1618}
1619
1620
1621/**
1622 * Verifies that our cached values of the VMCS controls are all
1623 * consistent with what's actually present in the VMCS.
1624 *
1625 * @returns VBox status code.
1626 * @param pVCpu The cross context virtual CPU structure.
1627 */
1628static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1629{
1630 uint32_t u32Val;
1631 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1632 AssertRCReturn(rc, rc);
1633 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1634 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1639 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1649 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1650
1651 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1652 {
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1656 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1657 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1658 }
1659
1660 return VINF_SUCCESS;
1661}
1662
1663
1664#ifdef VBOX_STRICT
1665/**
1666 * Verifies that our cached host EFER value has not changed
1667 * since we cached it.
1668 *
1669 * @param pVCpu The cross context virtual CPU structure.
1670 */
1671static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1672{
1673 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1674
1675 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1676 {
1677 uint64_t u64Val;
1678 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1679 AssertRC(rc);
1680
1681 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1682 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1683 }
1684}
1685
1686
1687/**
1688 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1689 * VMCS are correct.
1690 *
1691 * @param pVCpu The cross context virtual CPU structure.
1692 */
1693static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1694{
1695 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1696
1697 /* Verify MSR counts in the VMCS are what we think it should be. */
1698 uint32_t cMsrs;
1699 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1700 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1701
1702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1703 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1704
1705 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1709 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1710 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1711 {
1712 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1713 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1714 pGuestMsr->u32Msr, cMsrs));
1715
1716 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1717 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1718 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1719
1720 /* Verify that the permissions are as expected in the MSR bitmap. */
1721 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1722 {
1723 VMXMSREXITREAD enmRead;
1724 VMXMSREXITWRITE enmWrite;
1725 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1726 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1727 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1728 {
1729 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1730 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1731 }
1732 else
1733 {
1734 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1735 pGuestMsr->u32Msr, cMsrs));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1737 pGuestMsr->u32Msr, cMsrs));
1738 }
1739 }
1740 }
1741}
1742#endif /* VBOX_STRICT */
1743
1744
1745/**
1746 * Flushes the TLB using EPT.
1747 *
1748 * @returns VBox status code.
1749 * @param pVCpu The cross context virtual CPU structure of the calling
1750 * EMT. Can be NULL depending on @a enmFlush.
1751 * @param enmFlush Type of flush.
1752 *
1753 * @remarks Caller is responsible for making sure this function is called only
1754 * when NestedPaging is supported and providing @a enmFlush that is
1755 * supported by the CPU.
1756 * @remarks Can be called with interrupts disabled.
1757 */
1758static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1759{
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1762 au64Descriptor[0] = 0;
1763 else
1764 {
1765 Assert(pVCpu);
1766 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1767 }
1768 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1769
1770 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1771 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1772 rc));
1773 if ( RT_SUCCESS(rc)
1774 && pVCpu)
1775 {
1776 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1777 }
1778}
1779
1780
1781/**
1782 * Flushes the TLB using VPID.
1783 *
1784 * @returns VBox status code.
1785 * @param pVCpu The cross context virtual CPU structure of the calling
1786 * EMT. Can be NULL depending on @a enmFlush.
1787 * @param enmFlush Type of flush.
1788 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1789 * on @a enmFlush).
1790 *
1791 * @remarks Can be called with interrupts disabled.
1792 */
1793static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1794{
1795 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1796
1797 uint64_t au64Descriptor[2];
1798 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1799 {
1800 au64Descriptor[0] = 0;
1801 au64Descriptor[1] = 0;
1802 }
1803 else
1804 {
1805 AssertPtr(pVCpu);
1806 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1807 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1808 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1809 au64Descriptor[1] = GCPtr;
1810 }
1811
1812 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1813 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1814 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1815 if ( RT_SUCCESS(rc)
1816 && pVCpu)
1817 {
1818 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1819 }
1820 NOREF(rc);
1821}
1822
1823
1824/**
1825 * Invalidates a guest page by guest virtual address. Only relevant for
1826 * EPT/VPID, otherwise there is nothing really to invalidate.
1827 *
1828 * @returns VBox status code.
1829 * @param pVCpu The cross context virtual CPU structure.
1830 * @param GCVirt Guest virtual address of the page to invalidate.
1831 */
1832VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1833{
1834 AssertPtr(pVCpu);
1835 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1836
1837 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1838 if (!fFlushPending)
1839 {
1840 /*
1841 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1842 * the EPT case. See @bugref{6043} and @bugref{6177}.
1843 *
1844 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1845 * as this function maybe called in a loop with individual addresses.
1846 */
1847 PVM pVM = pVCpu->CTX_SUFF(pVM);
1848 if (pVM->hm.s.vmx.fVpid)
1849 {
1850 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1851
1852#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1853 /*
1854 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1855 * where executing INVVPID outside 64-bit mode does not flush translations of
1856 * 64-bit linear addresses, see @bugref{6208#c72}.
1857 */
1858 if (RT_HI_U32(GCVirt))
1859 fVpidFlush = false;
1860#endif
1861
1862 if (fVpidFlush)
1863 {
1864 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1880 * case where neither EPT nor VPID is supported by the CPU.
1881 *
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param pCpu Pointer to the global HM struct.
1884 *
1885 * @remarks Called with interrupts disabled.
1886 */
1887static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1888{
1889 AssertPtr(pVCpu);
1890 AssertPtr(pCpu);
1891
1892 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1893
1894 Assert(pCpu->idCpu != NIL_RTCPUID);
1895 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1896 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1897 pVCpu->hm.s.fForceTLBFlush = false;
1898 return;
1899}
1900
1901
1902/**
1903 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1904 *
1905 * @param pVCpu The cross context virtual CPU structure.
1906 * @param pCpu Pointer to the global HM CPU struct.
1907 *
1908 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1909 * nomenclature. The reason is, to avoid confusion in compare statements
1910 * since the host-CPU copies are named "ASID".
1911 *
1912 * @remarks Called with interrupts disabled.
1913 */
1914static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915{
1916#ifdef VBOX_WITH_STATISTICS
1917 bool fTlbFlushed = false;
1918# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1919# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1920 if (!fTlbFlushed) \
1921 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1922 } while (0)
1923#else
1924# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1925# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1926#endif
1927
1928 AssertPtr(pCpu);
1929 AssertPtr(pVCpu);
1930 Assert(pCpu->idCpu != NIL_RTCPUID);
1931
1932 PVM pVM = pVCpu->CTX_SUFF(pVM);
1933 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1934 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1935 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1939 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1940 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1941 * cannot reuse the current ASID anymore.
1942 */
1943 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1944 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1945 {
1946 ++pCpu->uCurrentAsid;
1947 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1948 {
1949 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1950 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1951 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1952 }
1953
1954 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 /*
1959 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1960 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1966 }
1967
1968 /* Check for explicit TLB flushes. */
1969 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1970 {
1971 /*
1972 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1973 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1974 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1975 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1976 * mappings, see @bugref{6568}.
1977 *
1978 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 }
1984
1985 pVCpu->hm.s.fForceTLBFlush = false;
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVCpu The cross context virtual CPU structure.
2011 * @param pCpu Pointer to the global HM CPU struct.
2012 *
2013 * @remarks Called with interrupts disabled.
2014 */
2015static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2016{
2017 AssertPtr(pVCpu);
2018 AssertPtr(pCpu);
2019 Assert(pCpu->idCpu != NIL_RTCPUID);
2020 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2021 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2022
2023 /*
2024 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2025 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2026 */
2027 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2028 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2029 {
2030 pVCpu->hm.s.fForceTLBFlush = true;
2031 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2032 }
2033
2034 /* Check for explicit TLB flushes. */
2035 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2039 }
2040
2041 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2042 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2043
2044 if (pVCpu->hm.s.fForceTLBFlush)
2045 {
2046 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2047 pVCpu->hm.s.fForceTLBFlush = false;
2048 }
2049}
2050
2051
2052/**
2053 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2054 *
2055 * @returns VBox status code.
2056 * @param pVCpu The cross context virtual CPU structure.
2057 * @param pCpu Pointer to the global HM CPU struct.
2058 *
2059 * @remarks Called with interrupts disabled.
2060 */
2061static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062{
2063 AssertPtr(pVCpu);
2064 AssertPtr(pCpu);
2065 Assert(pCpu->idCpu != NIL_RTCPUID);
2066 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2067 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2068
2069 /*
2070 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2071 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2072 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2073 * cannot reuse the current ASID anymore.
2074 */
2075 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2076 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2077 {
2078 pVCpu->hm.s.fForceTLBFlush = true;
2079 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2080 }
2081
2082 /* Check for explicit TLB flushes. */
2083 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2084 {
2085 /*
2086 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2087 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2088 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2089 * include fExplicitFlush's too) - an obscure corner case.
2090 */
2091 pVCpu->hm.s.fForceTLBFlush = true;
2092 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2093 }
2094
2095 PVM pVM = pVCpu->CTX_SUFF(pVM);
2096 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2097 if (pVCpu->hm.s.fForceTLBFlush)
2098 {
2099 ++pCpu->uCurrentAsid;
2100 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2101 {
2102 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2103 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2104 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2105 }
2106
2107 pVCpu->hm.s.fForceTLBFlush = false;
2108 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2109 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2110 if (pCpu->fFlushAsidBeforeUse)
2111 {
2112 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2113 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2114 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2115 {
2116 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2117 pCpu->fFlushAsidBeforeUse = false;
2118 }
2119 else
2120 {
2121 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2122 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2123 }
2124 }
2125 }
2126
2127 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2128 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2129 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2130 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2131 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2132 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2133 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2134
2135 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2136 AssertRC(rc);
2137}
2138
2139
2140/**
2141 * Flushes the guest TLB entry based on CPU capabilities.
2142 *
2143 * @param pVCpu The cross context virtual CPU structure.
2144 * @param pCpu Pointer to the global HM CPU struct.
2145 */
2146DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2147{
2148#ifdef HMVMX_ALWAYS_FLUSH_TLB
2149 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2150#endif
2151 PVM pVM = pVCpu->CTX_SUFF(pVM);
2152 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2153 {
2154 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2155 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2156 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2158 default:
2159 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2160 break;
2161 }
2162 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2163}
2164
2165
2166/**
2167 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2168 * TLB entries from the host TLB before VM-entry.
2169 *
2170 * @returns VBox status code.
2171 * @param pVM The cross context VM structure.
2172 */
2173static int hmR0VmxSetupTaggedTlb(PVM pVM)
2174{
2175 /*
2176 * Determine optimal flush type for Nested Paging.
2177 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2178 * guest execution (see hmR3InitFinalizeR0()).
2179 */
2180 if (pVM->hm.s.fNestedPaging)
2181 {
2182 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2185 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2186 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2187 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2188 else
2189 {
2190 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2192 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2193 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2194 }
2195
2196 /* Make sure the write-back cacheable memory type for EPT is supported. */
2197 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2198 {
2199 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2200 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2201 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2202 }
2203
2204 /* EPT requires a page-walk length of 4. */
2205 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2206 {
2207 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211 }
2212 else
2213 {
2214 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2215 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219 }
2220
2221 /*
2222 * Determine optimal flush type for VPID.
2223 */
2224 if (pVM->hm.s.vmx.fVpid)
2225 {
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2229 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2230 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2231 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2232 else
2233 {
2234 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2236 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2238 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2240 pVM->hm.s.vmx.fVpid = false;
2241 }
2242 }
2243 else
2244 {
2245 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2246 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2247 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2248 pVM->hm.s.vmx.fVpid = false;
2249 }
2250 }
2251
2252 /*
2253 * Setup the handler for flushing tagged-TLBs.
2254 */
2255 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2256 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2257 else if (pVM->hm.s.fNestedPaging)
2258 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2259 else if (pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2261 else
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Sets up pin-based VM-execution controls in the VMCS.
2269 *
2270 * @returns VBox status code.
2271 * @param pVCpu The cross context virtual CPU structure.
2272 *
2273 * @remarks We don't really care about optimizing vmwrites here as it's done only
2274 * once per VM and hence we don't care about VMCS-field cache comparisons.
2275 */
2276static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2277{
2278 PVM pVM = pVCpu->CTX_SUFF(pVM);
2279 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2280 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2281
2282 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2283 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2284
2285 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2286 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2287
2288 /* Enable the VMX preemption timer. */
2289 if (pVM->hm.s.vmx.fUsePreemptTimer)
2290 {
2291 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2293 }
2294
2295#if 0
2296 /* Enable posted-interrupt processing. */
2297 if (pVM->hm.s.fPostedIntrs)
2298 {
2299 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2300 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2301 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2302 }
2303#endif
2304
2305 if ((fVal & fZap) != fVal)
2306 {
2307 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2308 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2309 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2310 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2311 }
2312
2313 /* Commit it to the VMCS and update our cache. */
2314 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2315 AssertRCReturn(rc, rc);
2316 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2317
2318 return VINF_SUCCESS;
2319}
2320
2321
2322/**
2323 * Sets up secondary processor-based VM-execution controls in the VMCS.
2324 *
2325 * @returns VBox status code.
2326 * @param pVCpu The cross context virtual CPU structure.
2327 *
2328 * @remarks We don't really care about optimizing vmwrites here as it's done only
2329 * once per VM and hence we don't care about VMCS-field cache comparisons.
2330 */
2331static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2332{
2333 PVM pVM = pVCpu->CTX_SUFF(pVM);
2334 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2335 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2336
2337 /* WBINVD causes a VM-exit. */
2338 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2339 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2340
2341 /* Enable EPT (aka nested-paging). */
2342 if (pVM->hm.s.fNestedPaging)
2343 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2344
2345 /*
2346 * Enable the INVPCID instruction if supported by the hardware and we expose
2347 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2348 */
2349 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2350 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2351 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2352
2353 /* Enable VPID. */
2354 if (pVM->hm.s.vmx.fVpid)
2355 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2356
2357 /* Enable Unrestricted guest execution. */
2358 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2359 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2360
2361#if 0
2362 if (pVM->hm.s.fVirtApicRegs)
2363 {
2364 /* Enable APIC-register virtualization. */
2365 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2366 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2367
2368 /* Enable virtual-interrupt delivery. */
2369 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2371 }
2372#endif
2373
2374 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2375 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2376 * done dynamically. */
2377 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2378 {
2379 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2380 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2381 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2382 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2383 AssertRCReturn(rc, rc);
2384 }
2385
2386 /* Enable RDTSCP. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2388 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2389
2390 /* Enable Pause-Loop exiting. */
2391 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2392 && pVM->hm.s.vmx.cPleGapTicks
2393 && pVM->hm.s.vmx.cPleWindowTicks)
2394 {
2395 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2396
2397 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2398 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2399 AssertRCReturn(rc, rc);
2400 }
2401
2402 if ((fVal & fZap) != fVal)
2403 {
2404 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2405 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2406 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2407 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2408 }
2409
2410 /* Commit it to the VMCS and update our cache. */
2411 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2412 AssertRCReturn(rc, rc);
2413 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2414
2415 return VINF_SUCCESS;
2416}
2417
2418
2419/**
2420 * Sets up processor-based VM-execution controls in the VMCS.
2421 *
2422 * @returns VBox status code.
2423 * @param pVCpu The cross context virtual CPU structure.
2424 *
2425 * @remarks We don't really care about optimizing vmwrites here as it's done only
2426 * once per VM and hence we don't care about VMCS-field cache comparisons.
2427 */
2428static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2429{
2430 PVM pVM = pVCpu->CTX_SUFF(pVM);
2431 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2432 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2433
2434 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2435 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2436 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2437 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2438 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2441
2442 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2443 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2444 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2445 {
2446 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2447 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2448 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2449 }
2450
2451 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2452 if (!pVM->hm.s.fNestedPaging)
2453 {
2454 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2455 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2456 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2458 }
2459
2460 /* Use TPR shadowing if supported by the CPU. */
2461 if ( PDMHasApic(pVM)
2462 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2463 {
2464 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2465 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2466 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2467 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2468 AssertRCReturn(rc, rc);
2469
2470 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2471 /* CR8 writes cause a VM-exit based on TPR threshold. */
2472 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2473 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2474 }
2475 else
2476 {
2477 /*
2478 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2479 * Set this control only for 64-bit guests.
2480 */
2481 if (pVM->hm.s.fAllow64BitGuests)
2482 {
2483 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2484 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2485 }
2486 }
2487
2488 /* Use MSR-bitmaps if supported by the CPU. */
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2490 {
2491 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2492
2493 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2494 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2495 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 AssertRCReturn(rc, rc);
2497
2498 /*
2499 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2500 * automatically using dedicated fields in the VMCS.
2501 */
2502 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2503 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507#if HC_ARCH_BITS == 64
2508 /*
2509 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2510 */
2511 if (pVM->hm.s.fAllow64BitGuests)
2512 {
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 }
2518#endif
2519 /*
2520 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2521 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2522 */
2523 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525
2526 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2527 }
2528
2529 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2530 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2531 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2532
2533 if ((fVal & fZap) != fVal)
2534 {
2535 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2536 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2537 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2538 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2539 }
2540
2541 /* Commit it to the VMCS and update our cache. */
2542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2543 AssertRCReturn(rc, rc);
2544 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2545
2546 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2547 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 return hmR0VmxSetupProcCtls2(pVCpu);
2549
2550 /* Sanity check, should not really happen. */
2551 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2552 {
2553 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2559 return VINF_SUCCESS;
2560}
2561
2562
2563/**
2564 * Sets up miscellaneous (everything other than Pin & Processor-based
2565 * VM-execution) control fields in the VMCS.
2566 *
2567 * @returns VBox status code.
2568 * @param pVCpu The cross context virtual CPU structure.
2569 */
2570static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2571{
2572 AssertPtr(pVCpu);
2573
2574 int rc = VERR_GENERAL_FAILURE;
2575
2576 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2577#if 0
2578 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2580 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2581
2582 /*
2583 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2584 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2585 * We thus use the exception bitmap to control it rather than use both.
2586 */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2589
2590 /* All IO & IOIO instructions cause VM-exits. */
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2592 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2593
2594 /* Initialize the MSR-bitmap area. */
2595 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2598 AssertRCReturn(rc, rc);
2599#endif
2600
2601 /* Setup MSR auto-load/store area. */
2602 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2606 AssertRCReturn(rc, rc);
2607
2608 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2609 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2610 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2615 AssertRCReturn(rc, rc);
2616
2617 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2618#if 0
2619 /* Setup debug controls */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2621 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2622 AssertRCReturn(rc, rc);
2623#endif
2624
2625 return rc;
2626}
2627
2628
2629/**
2630 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2631 *
2632 * We shall setup those exception intercepts that don't change during the
2633 * lifetime of the VM here. The rest are done dynamically while loading the
2634 * guest state.
2635 *
2636 * @returns VBox status code.
2637 * @param pVCpu The cross context virtual CPU structure.
2638 */
2639static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2640{
2641 AssertPtr(pVCpu);
2642
2643 uint32_t u32XcptBitmap;
2644
2645 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2646 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2647
2648 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2649 and writes, and because recursive #DBs can cause the CPU hang, we must always
2650 intercept #DB. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2652
2653 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2654 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2655 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2656
2657 /* Commit it to the VMCS. */
2658 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2659 AssertRCReturn(rc, rc);
2660
2661 /* Update our cache of the exception bitmap. */
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 return VINF_SUCCESS;
2664}
2665
2666
2667/**
2668 * Does per-VM VT-x initialization.
2669 *
2670 * @returns VBox status code.
2671 * @param pVM The cross context VM structure.
2672 */
2673VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2674{
2675 LogFlowFunc(("pVM=%p\n", pVM));
2676
2677 int rc = hmR0VmxStructsAlloc(pVM);
2678 if (RT_FAILURE(rc))
2679 {
2680 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2681 return rc;
2682 }
2683
2684 return VINF_SUCCESS;
2685}
2686
2687
2688/**
2689 * Does per-VM VT-x termination.
2690 *
2691 * @returns VBox status code.
2692 * @param pVM The cross context VM structure.
2693 */
2694VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2695{
2696 LogFlowFunc(("pVM=%p\n", pVM));
2697
2698#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2700 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2701#endif
2702 hmR0VmxStructsFree(pVM);
2703 return VINF_SUCCESS;
2704}
2705
2706
2707/**
2708 * Sets up the VM for execution under VT-x.
2709 * This function is only called once per-VM during initialization.
2710 *
2711 * @returns VBox status code.
2712 * @param pVM The cross context VM structure.
2713 */
2714VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2715{
2716 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2717 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2718
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721 /*
2722 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2723 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2724 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2725 */
2726 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2727 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2728 || !pVM->hm.s.vmx.pRealModeTSS))
2729 {
2730 LogRelFunc(("Invalid real-on-v86 state.\n"));
2731 return VERR_INTERNAL_ERROR;
2732 }
2733
2734 /* Initialize these always, see hmR3InitFinalizeR0().*/
2735 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2736 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2737
2738 /* Setup the tagged-TLB flush handlers. */
2739 int rc = hmR0VmxSetupTaggedTlb(pVM);
2740 if (RT_FAILURE(rc))
2741 {
2742 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2743 return rc;
2744 }
2745
2746 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2747 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2748#if HC_ARCH_BITS == 64
2749 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2750 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2751 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2752 {
2753 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2754 }
2755#endif
2756
2757 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2758 RTCCUINTREG uHostCR4 = ASMGetCR4();
2759 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2760 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2761
2762 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2763 {
2764 PVMCPU pVCpu = &pVM->aCpus[i];
2765 AssertPtr(pVCpu);
2766 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2767
2768 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2769 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2770
2771 /* Set revision dword at the beginning of the VMCS structure. */
2772 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2773
2774 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2775 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2777 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2778
2779 /* Load this VMCS as the current VMCS. */
2780 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2781 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2782 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2783
2784 rc = hmR0VmxSetupPinCtls(pVCpu);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2786 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2787
2788 rc = hmR0VmxSetupProcCtls(pVCpu);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2790 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupMiscCtls(pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 rc = hmR0VmxInitXcptBitmap(pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800#if HC_ARCH_BITS == 32
2801 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804#endif
2805
2806 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2807 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2812
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2814 }
2815
2816 return VINF_SUCCESS;
2817}
2818
2819
2820/**
2821 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2822 * the VMCS.
2823 *
2824 * @returns VBox status code.
2825 */
2826static int hmR0VmxExportHostControlRegs(void)
2827{
2828 RTCCUINTREG uReg = ASMGetCR0();
2829 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2830 AssertRCReturn(rc, rc);
2831
2832 uReg = ASMGetCR3();
2833 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2834 AssertRCReturn(rc, rc);
2835
2836 uReg = ASMGetCR4();
2837 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2838 AssertRCReturn(rc, rc);
2839 return rc;
2840}
2841
2842
2843/**
2844 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2845 * the host-state area in the VMCS.
2846 *
2847 * @returns VBox status code.
2848 * @param pVCpu The cross context virtual CPU structure.
2849 */
2850static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2851{
2852#if HC_ARCH_BITS == 64
2853/**
2854 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2855 * requirements. See hmR0VmxExportHostSegmentRegs().
2856 */
2857# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2858 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2859 { \
2860 bool fValidSelector = true; \
2861 if ((selValue) & X86_SEL_LDT) \
2862 { \
2863 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2864 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2865 } \
2866 if (fValidSelector) \
2867 { \
2868 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2869 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2870 } \
2871 (selValue) = 0; \
2872 }
2873
2874 /*
2875 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2876 * should -not- save the messed up state without restoring the original host-state,
2877 * see @bugref{7240}.
2878 *
2879 * This apparently can happen (most likely the FPU changes), deal with it rather than
2880 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2881 */
2882 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2883 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2884 {
2885 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2886 pVCpu->idCpu));
2887 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2888 }
2889 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2890#else
2891 RT_NOREF(pVCpu);
2892#endif
2893
2894 /*
2895 * Host DS, ES, FS and GS segment registers.
2896 */
2897#if HC_ARCH_BITS == 64
2898 RTSEL uSelDS = ASMGetDS();
2899 RTSEL uSelES = ASMGetES();
2900 RTSEL uSelFS = ASMGetFS();
2901 RTSEL uSelGS = ASMGetGS();
2902#else
2903 RTSEL uSelDS = 0;
2904 RTSEL uSelES = 0;
2905 RTSEL uSelFS = 0;
2906 RTSEL uSelGS = 0;
2907#endif
2908
2909 /*
2910 * Host CS and SS segment registers.
2911 */
2912 RTSEL uSelCS = ASMGetCS();
2913 RTSEL uSelSS = ASMGetSS();
2914
2915 /*
2916 * Host TR segment register.
2917 */
2918 RTSEL uSelTR = ASMGetTR();
2919
2920#if HC_ARCH_BITS == 64
2921 /*
2922 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2923 * gain VM-entry and restore them before we get preempted.
2924 *
2925 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2926 */
2927 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2928 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2929 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2930 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2931# undef VMXLOCAL_ADJUST_HOST_SEG
2932#endif
2933
2934 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2935 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2936 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2937 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2938 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2939 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2940 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2941 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2942 Assert(uSelCS);
2943 Assert(uSelTR);
2944
2945 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2946#if 0
2947 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2948 Assert(uSelSS != 0);
2949#endif
2950
2951 /* Write these host selector fields into the host-state area in the VMCS. */
2952 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2953 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2954#if HC_ARCH_BITS == 64
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2956 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2959#else
2960 NOREF(uSelDS);
2961 NOREF(uSelES);
2962 NOREF(uSelFS);
2963 NOREF(uSelGS);
2964#endif
2965 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2966 AssertRCReturn(rc, rc);
2967
2968 /*
2969 * Host GDTR and IDTR.
2970 */
2971 RTGDTR Gdtr;
2972 RTIDTR Idtr;
2973 RT_ZERO(Gdtr);
2974 RT_ZERO(Idtr);
2975 ASMGetGDTR(&Gdtr);
2976 ASMGetIDTR(&Idtr);
2977 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2978 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2979 AssertRCReturn(rc, rc);
2980
2981#if HC_ARCH_BITS == 64
2982 /*
2983 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2984 * them to the maximum limit (0xffff) on every VM-exit.
2985 */
2986 if (Gdtr.cbGdt != 0xffff)
2987 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2988
2989 /*
2990 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2991 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2992 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2993 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2994 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2995 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2996 * at 0xffff on hosts where we are sure it won't cause trouble.
2997 */
2998# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2999 if (Idtr.cbIdt < 0x0fff)
3000# else
3001 if (Idtr.cbIdt != 0xffff)
3002# endif
3003 {
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3005 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3006 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3007 }
3008#endif
3009
3010 /*
3011 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3012 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3013 * RPL should be too in most cases.
3014 */
3015 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3016 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3017
3018 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3019#if HC_ARCH_BITS == 64
3020 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3021
3022 /*
3023 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3024 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3025 * restoration if the host has something else. Task switching is not supported in 64-bit
3026 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3027 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3028 *
3029 * [1] See Intel spec. 3.5 "System Descriptor Types".
3030 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3031 */
3032 PVM pVM = pVCpu->CTX_SUFF(pVM);
3033 Assert(pDesc->System.u4Type == 11);
3034 if ( pDesc->System.u16LimitLow != 0x67
3035 || pDesc->System.u4LimitHigh)
3036 {
3037 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3038 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3039 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3041 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3042 }
3043
3044 /*
3045 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3046 */
3047 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3048 {
3049 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3050 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3051 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3052 {
3053 /* The GDT is read-only but the writable GDT is available. */
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3055 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3056 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3057 AssertRCReturn(rc, rc);
3058 }
3059 }
3060#else
3061 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3062#endif
3063 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3064 AssertRCReturn(rc, rc);
3065
3066 /*
3067 * Host FS base and GS base.
3068 */
3069#if HC_ARCH_BITS == 64
3070 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3071 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3072 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3073 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3074 AssertRCReturn(rc, rc);
3075
3076 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3077 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3078 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3079 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3080 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3081#endif
3082 return VINF_SUCCESS;
3083}
3084
3085
3086/**
3087 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3088 * host-state area of the VMCS.
3089 *
3090 * Theses MSRs will be automatically restored on the host after every successful
3091 * VM-exit.
3092 *
3093 * @returns VBox status code.
3094 * @param pVCpu The cross context virtual CPU structure.
3095 *
3096 * @remarks No-long-jump zone!!!
3097 */
3098static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3099{
3100 AssertPtr(pVCpu);
3101 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3102
3103 /*
3104 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3105 * rather than swapping them on every VM-entry.
3106 */
3107 hmR0VmxLazySaveHostMsrs(pVCpu);
3108
3109 /*
3110 * Host Sysenter MSRs.
3111 */
3112 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3113#if HC_ARCH_BITS == 32
3114 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3115 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3116#else
3117 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3118 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3119#endif
3120 AssertRCReturn(rc, rc);
3121
3122 /*
3123 * Host EFER MSR.
3124 *
3125 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3126 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3127 */
3128 PVM pVM = pVCpu->CTX_SUFF(pVM);
3129 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3130 {
3131 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3132 AssertRCReturn(rc, rc);
3133 }
3134
3135 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3136
3137 return VINF_SUCCESS;
3138}
3139
3140
3141/**
3142 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3143 *
3144 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3145 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3146 * hmR0VMxExportGuestEntryCtls().
3147 *
3148 * @returns true if we need to load guest EFER, false otherwise.
3149 * @param pVCpu The cross context virtual CPU structure.
3150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3151 * out-of-sync. Make sure to update the required fields
3152 * before using them.
3153 *
3154 * @remarks Requires EFER, CR4.
3155 * @remarks No-long-jump zone!!!
3156 */
3157static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3158{
3159#ifdef HMVMX_ALWAYS_SWAP_EFER
3160 return true;
3161#endif
3162
3163#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3164 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3165 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3166 return false;
3167#endif
3168
3169 PVM pVM = pVCpu->CTX_SUFF(pVM);
3170 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3171 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3172
3173 /*
3174 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3175 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3176 */
3177 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3178 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3179 {
3180 return true;
3181 }
3182
3183 /*
3184 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3185 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3186 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3187 */
3188 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3189 && (pMixedCtx->cr0 & X86_CR0_PG)
3190 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3191 {
3192 /* Assert that host is PAE capable. */
3193 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3194 return true;
3195 }
3196
3197 return false;
3198}
3199
3200
3201/**
3202 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3203 *
3204 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3205 * see Intel spec. 24.8.1 "VM-entry controls".
3206 *
3207 * @returns VBox status code.
3208 * @param pVCpu The cross context virtual CPU structure.
3209 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3210 * out-of-sync. Make sure to update the required fields
3211 * before using them.
3212 *
3213 * @remarks Requires EFER.
3214 * @remarks No-long-jump zone!!!
3215 */
3216static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3217{
3218 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3219 {
3220 PVM pVM = pVCpu->CTX_SUFF(pVM);
3221 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3222 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3223
3224 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3225 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3226
3227 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3228 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3229 {
3230 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3231 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3232 }
3233 else
3234 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3235
3236 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3237 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3238 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3239 {
3240 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3241 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3242 }
3243
3244 /*
3245 * The following should -not- be set (since we're not in SMM mode):
3246 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3247 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3248 */
3249
3250 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3251 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3252
3253 if ((fVal & fZap) != fVal)
3254 {
3255 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3256 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3257 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3259 }
3260
3261 /* Commit it to the VMCS and update our cache. */
3262 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3263 {
3264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3265 AssertRCReturn(rc, rc);
3266 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3267 }
3268
3269 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3270 }
3271 return VINF_SUCCESS;
3272}
3273
3274
3275/**
3276 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3277 *
3278 * @returns VBox status code.
3279 * @param pVCpu The cross context virtual CPU structure.
3280 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3281 * out-of-sync. Make sure to update the required fields
3282 * before using them.
3283 *
3284 * @remarks Requires EFER.
3285 */
3286static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3287{
3288 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3289 {
3290 PVM pVM = pVCpu->CTX_SUFF(pVM);
3291 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3292 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3293
3294 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3295 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3296
3297 /*
3298 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3299 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3300 * hmR0VmxExportHostMsrs().
3301 */
3302#if HC_ARCH_BITS == 64
3303 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3304 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3305#else
3306 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3307 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3308 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3309 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3310 {
3311 /* The switcher returns to long mode, EFER is managed by the switcher. */
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314 }
3315 else
3316 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3317#endif
3318
3319 /* If the newer VMCS fields for managing EFER exists, use it. */
3320 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322 {
3323 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3324 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3325 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3326 }
3327
3328 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3329 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3330
3331 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3332 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3333 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3334
3335 /* Enable saving of the VMX preemption timer value on VM-exit. */
3336 if ( pVM->hm.s.vmx.fUsePreemptTimer
3337 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3338 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3339
3340 if ((fVal & fZap) != fVal)
3341 {
3342 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3343 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3344 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3345 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3346 }
3347
3348 /* Commit it to the VMCS and update our cache. */
3349 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3350 {
3351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3352 AssertRCReturn(rc, rc);
3353 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3354 }
3355
3356 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3357 }
3358 return VINF_SUCCESS;
3359}
3360
3361
3362/**
3363 * Sets the TPR threshold in the VMCS.
3364 *
3365 * @returns VBox status code.
3366 * @param pVCpu The cross context virtual CPU structure.
3367 * @param u32TprThreshold The TPR threshold (task-priority class only).
3368 */
3369DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3370{
3371 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3372 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3373 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3374}
3375
3376
3377/**
3378 * Exports the guest APIC TPR state into the VMCS.
3379 *
3380 * @returns VBox status code.
3381 * @param pVCpu The cross context virtual CPU structure.
3382 *
3383 * @remarks No-long-jump zone!!!
3384 */
3385static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3386{
3387 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3388 {
3389 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3390 && APICIsEnabled(pVCpu))
3391 {
3392 /*
3393 * Setup TPR shadowing.
3394 */
3395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3396 {
3397 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3398
3399 bool fPendingIntr = false;
3400 uint8_t u8Tpr = 0;
3401 uint8_t u8PendingIntr = 0;
3402 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3403 AssertRCReturn(rc, rc);
3404
3405 /*
3406 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3407 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3408 * priority of the pending interrupt so we can deliver the interrupt. If there
3409 * are no interrupts pending, set threshold to 0 to not cause any
3410 * TPR-below-threshold VM-exits.
3411 */
3412 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3413 uint32_t u32TprThreshold = 0;
3414 if (fPendingIntr)
3415 {
3416 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3417 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3418 const uint8_t u8TprPriority = u8Tpr >> 4;
3419 if (u8PendingPriority <= u8TprPriority)
3420 u32TprThreshold = u8PendingPriority;
3421 }
3422
3423 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3424 AssertRCReturn(rc, rc);
3425 }
3426 }
3427 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3428 }
3429 return VINF_SUCCESS;
3430}
3431
3432
3433/**
3434 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3435 *
3436 * @returns Guest's interruptibility-state.
3437 * @param pVCpu The cross context virtual CPU structure.
3438 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3439 * out-of-sync. Make sure to update the required fields
3440 * before using them.
3441 *
3442 * @remarks No-long-jump zone!!!
3443 */
3444static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3445{
3446 /*
3447 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3448 */
3449 uint32_t fIntrState = 0;
3450 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3451 {
3452 /* If inhibition is active, RIP & RFLAGS should've been accessed
3453 (i.e. read previously from the VMCS or from ring-3). */
3454#ifdef VBOX_STRICT
3455 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3456 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3457#endif
3458 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3459 {
3460 if (pMixedCtx->eflags.Bits.u1IF)
3461 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3462 else
3463 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3464 }
3465 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466 {
3467 /*
3468 * We can clear the inhibit force flag as even if we go back to the recompiler
3469 * without executing guest code in VT-x, the flag's condition to be cleared is
3470 * met and thus the cleared state is correct.
3471 */
3472 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3473 }
3474 }
3475
3476 /*
3477 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3478 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3479 * setting this would block host-NMIs and IRET will not clear the blocking.
3480 *
3481 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3482 */
3483 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3484 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3485 {
3486 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3487 }
3488
3489 return fIntrState;
3490}
3491
3492
3493/**
3494 * Exports the guest's interruptibility-state into the guest-state area in the
3495 * VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 * @param fIntrState The interruptibility-state to set.
3500 */
3501static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3502{
3503 NOREF(pVCpu);
3504 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3505 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3506 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3507}
3508
3509
3510/**
3511 * Exports the exception intercepts required for guest execution in the VMCS.
3512 *
3513 * @returns VBox status code.
3514 * @param pVCpu The cross context virtual CPU structure.
3515 *
3516 * @remarks No-long-jump zone!!!
3517 */
3518static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3519{
3520 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3521 {
3522 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3523 if (pVCpu->hm.s.fGIMTrapXcptUD)
3524 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3525#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3526 else
3527 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3528#endif
3529
3530 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3531 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3532
3533 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3534 AssertRCReturn(rc, rc);
3535
3536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3537 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3538 }
3539 return VINF_SUCCESS;
3540}
3541
3542
3543/**
3544 * Exports the guest's RIP into the guest-state area in the VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3549 * out-of-sync. Make sure to update the required fields
3550 * before using them.
3551 *
3552 * @remarks No-long-jump zone!!!
3553 */
3554static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3555{
3556 int rc = VINF_SUCCESS;
3557 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3558 {
3559 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3560 AssertRCReturn(rc, rc);
3561
3562 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3564 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3565 else
3566 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3567
3568 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3569 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3570 }
3571 return rc;
3572}
3573
3574
3575/**
3576 * Exports the guest's RSP into the guest-state area in the VMCS.
3577 *
3578 * @returns VBox status code.
3579 * @param pVCpu The cross context virtual CPU structure.
3580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3581 * out-of-sync. Make sure to update the required fields
3582 * before using them.
3583 *
3584 * @remarks No-long-jump zone!!!
3585 */
3586static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3587{
3588 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3589 {
3590 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3591 AssertRCReturn(rc, rc);
3592
3593 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3594 }
3595 return VINF_SUCCESS;
3596}
3597
3598
3599/**
3600 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3601 *
3602 * @returns VBox status code.
3603 * @param pVCpu The cross context virtual CPU structure.
3604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3605 * out-of-sync. Make sure to update the required fields
3606 * before using them.
3607 *
3608 * @remarks No-long-jump zone!!!
3609 */
3610static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3611{
3612 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613 {
3614 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3615 Let us assert it as such and use 32-bit VMWRITE. */
3616 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3617 X86EFLAGS fEFlags = pMixedCtx->eflags;
3618 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3619 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3620
3621 /*
3622 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3623 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3624 * can run the real-mode guest code under Virtual 8086 mode.
3625 */
3626 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3627 {
3628 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3629 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3630 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3631 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3632 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3633 }
3634
3635 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3636 AssertRCReturn(rc, rc);
3637
3638 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3639 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3640 }
3641 return VINF_SUCCESS;
3642}
3643
3644
3645/**
3646 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3647 *
3648 * The guest FPU state is always pre-loaded hence we don't need to bother about
3649 * sharing FPU related CR0 bits between the guest and host.
3650 *
3651 * @returns VBox status code.
3652 * @param pVCpu The cross context virtual CPU structure.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3660{
3661 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3662 {
3663 PVM pVM = pVCpu->CTX_SUFF(pVM);
3664 Assert(!RT_HI_U32(pMixedCtx->cr0));
3665 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3666 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3667
3668 /*
3669 * Setup VT-x's view of the guest CR0.
3670 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3671 */
3672 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3673 if (pVM->hm.s.fNestedPaging)
3674 {
3675 if (CPUMIsGuestPagingEnabled(pVCpu))
3676 {
3677 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3678 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3679 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3680 }
3681 else
3682 {
3683 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3684 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3685 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3686 }
3687
3688 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3689 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3690 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692 else
3693 {
3694 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3695 u32GuestCr0 |= X86_CR0_WP;
3696 }
3697
3698 /*
3699 * Guest FPU bits.
3700 *
3701 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3702 * using CR0.TS.
3703 *
3704 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3705 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3706 */
3707 u32GuestCr0 |= X86_CR0_NE;
3708
3709 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3710 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3711
3712 /*
3713 * Update exception intercepts.
3714 */
3715 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3716 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3717 {
3718 Assert(PDMVmmDevHeapIsEnabled(pVM));
3719 Assert(pVM->hm.s.vmx.pRealModeTSS);
3720 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3721 }
3722 else
3723 {
3724 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3725 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3726 if (fInterceptMF)
3727 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3728 }
3729
3730 /* Additional intercepts for debugging, define these yourself explicitly. */
3731#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3732 uXcptBitmap |= 0
3733 | RT_BIT(X86_XCPT_BP)
3734 | RT_BIT(X86_XCPT_DE)
3735 | RT_BIT(X86_XCPT_NM)
3736 | RT_BIT(X86_XCPT_TS)
3737 | RT_BIT(X86_XCPT_UD)
3738 | RT_BIT(X86_XCPT_NP)
3739 | RT_BIT(X86_XCPT_SS)
3740 | RT_BIT(X86_XCPT_GP)
3741 | RT_BIT(X86_XCPT_PF)
3742 | RT_BIT(X86_XCPT_MF)
3743 ;
3744#elif defined(HMVMX_ALWAYS_TRAP_PF)
3745 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3746#endif
3747 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3748 {
3749 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3751 }
3752 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 AssertRCReturn(rc, rc);
3798
3799 /* Update our caches. */
3800 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3801 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3802
3803 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3804
3805 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3806 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3807 }
3808
3809 return VINF_SUCCESS;
3810}
3811
3812
3813/**
3814 * Exports the guest control registers (CR3, CR4) into the guest-state area
3815 * in the VMCS.
3816 *
3817 * @returns VBox strict status code.
3818 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3819 * without unrestricted guest access and the VMMDev is not presently
3820 * mapped (e.g. EFI32).
3821 *
3822 * @param pVCpu The cross context virtual CPU structure.
3823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3824 * out-of-sync. Make sure to update the required fields
3825 * before using them.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3845 if (pVM->hm.s.fNestedPaging)
3846 {
3847 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3848
3849 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3850 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3851 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3853
3854 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3855 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3856 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3857
3858 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3859 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3860 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3861 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3863 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3864 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3865
3866 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3867 AssertRCReturn(rc, rc);
3868
3869 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3870 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3871 {
3872 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3873 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3874 {
3875 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3876 AssertRCReturn(rc, rc);
3877 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3878 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3881 AssertRCReturn(rc, rc);
3882 }
3883
3884 /*
3885 * The guest's view of its CR3 is unblemished with Nested Paging when the
3886 * guest is using paging or we have unrestricted guest execution to handle
3887 * the guest when it's not using paging.
3888 */
3889 GCPhysGuestCR3 = pMixedCtx->cr3;
3890 }
3891 else
3892 {
3893 /*
3894 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3895 * thinks it accesses physical memory directly, we use our identity-mapped
3896 * page table to map guest-linear to guest-physical addresses. EPT takes care
3897 * of translating it to host-physical addresses.
3898 */
3899 RTGCPHYS GCPhys;
3900 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3901
3902 /* We obtain it here every time as the guest could have relocated this PCI region. */
3903 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3904 if (RT_SUCCESS(rc))
3905 { /* likely */ }
3906 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3907 {
3908 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3909 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3910 }
3911 else
3912 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3913
3914 GCPhysGuestCR3 = GCPhys;
3915 }
3916
3917 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3918 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3919 AssertRCReturn(rc, rc);
3920 }
3921 else
3922 {
3923 /* Non-nested paging case, just use the hypervisor's CR3. */
3924 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3925
3926 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3927 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3928 AssertRCReturn(rc, rc);
3929 }
3930
3931 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3932 }
3933
3934 /*
3935 * Guest CR4.
3936 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3937 */
3938 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3939 {
3940 Assert(!RT_HI_U32(pMixedCtx->cr4));
3941 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3942 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3943
3944 /*
3945 * Setup VT-x's view of the guest CR4.
3946 *
3947 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3948 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3949 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3950 *
3951 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3952 */
3953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3954 {
3955 Assert(pVM->hm.s.vmx.pRealModeTSS);
3956 Assert(PDMVmmDevHeapIsEnabled(pVM));
3957 u32GuestCr4 &= ~X86_CR4_VME;
3958 }
3959
3960 if (pVM->hm.s.fNestedPaging)
3961 {
3962 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3963 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3964 {
3965 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3966 u32GuestCr4 |= X86_CR4_PSE;
3967 /* Our identity mapping is a 32-bit page directory. */
3968 u32GuestCr4 &= ~X86_CR4_PAE;
3969 }
3970 /* else use guest CR4.*/
3971 }
3972 else
3973 {
3974 /*
3975 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3976 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3977 */
3978 switch (pVCpu->hm.s.enmShadowMode)
3979 {
3980 case PGMMODE_REAL: /* Real-mode. */
3981 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3982 case PGMMODE_32_BIT: /* 32-bit paging. */
3983 {
3984 u32GuestCr4 &= ~X86_CR4_PAE;
3985 break;
3986 }
3987
3988 case PGMMODE_PAE: /* PAE paging. */
3989 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3990 {
3991 u32GuestCr4 |= X86_CR4_PAE;
3992 break;
3993 }
3994
3995 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3996 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3997#ifdef VBOX_ENABLE_64_BITS_GUESTS
3998 break;
3999#endif
4000 default:
4001 AssertFailed();
4002 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4003 }
4004 }
4005
4006 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4007 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4008 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4009 u32GuestCr4 |= fSetCr4;
4010 u32GuestCr4 &= fZapCr4;
4011
4012 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4013 that would cause a VM-exit. */
4014 uint32_t u32Cr4Mask = X86_CR4_VME
4015 | X86_CR4_PAE
4016 | X86_CR4_PGE
4017 | X86_CR4_PSE
4018 | X86_CR4_VMXE;
4019 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4020 u32Cr4Mask |= X86_CR4_OSXSAVE;
4021 if (pVM->cpum.ro.GuestFeatures.fPcid)
4022 u32Cr4Mask |= X86_CR4_PCIDE;
4023
4024 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4025 into the VMCS and update our cache. */
4026 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4027 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4028 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4029 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4030 AssertRCReturn(rc, rc);
4031 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4032
4033 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4034 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4035
4036 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4037
4038 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4039 fZapCr4));
4040 }
4041 return rc;
4042}
4043
4044
4045/**
4046 * Exports the guest debug registers into the guest-state area in the VMCS.
4047 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4048 *
4049 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4050 *
4051 * @returns VBox status code.
4052 * @param pVCpu The cross context virtual CPU structure.
4053 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4054 * out-of-sync. Make sure to update the required fields
4055 * before using them.
4056 *
4057 * @remarks No-long-jump zone!!!
4058 */
4059static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4060{
4061 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4062
4063#ifdef VBOX_STRICT
4064 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4065 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4066 {
4067 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4068 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4069 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4070 }
4071#endif
4072
4073 bool fSteppingDB = false;
4074 bool fInterceptMovDRx = false;
4075 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4076 if (pVCpu->hm.s.fSingleInstruction)
4077 {
4078 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4079 PVM pVM = pVCpu->CTX_SUFF(pVM);
4080 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4081 {
4082 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4083 Assert(fSteppingDB == false);
4084 }
4085 else
4086 {
4087 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4088 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4089 pVCpu->hm.s.fClearTrapFlag = true;
4090 fSteppingDB = true;
4091 }
4092 }
4093
4094 uint32_t u32GuestDr7;
4095 if ( fSteppingDB
4096 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4097 {
4098 /*
4099 * Use the combined guest and host DRx values found in the hypervisor register set
4100 * because the debugger has breakpoints active or someone is single stepping on the
4101 * host side without a monitor trap flag.
4102 *
4103 * Note! DBGF expects a clean DR6 state before executing guest code.
4104 */
4105#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4106 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4107 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4108 {
4109 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4110 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4111 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4112 }
4113 else
4114#endif
4115 if (!CPUMIsHyperDebugStateActive(pVCpu))
4116 {
4117 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4118 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4119 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4120 }
4121
4122 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4123 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4124 pVCpu->hm.s.fUsingHyperDR7 = true;
4125 fInterceptMovDRx = true;
4126 }
4127 else
4128 {
4129 /*
4130 * If the guest has enabled debug registers, we need to load them prior to
4131 * executing guest code so they'll trigger at the right time.
4132 */
4133 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4134 {
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4142 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4143 }
4144 else
4145#endif
4146 if (!CPUMIsGuestDebugStateActive(pVCpu))
4147 {
4148 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4149 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4150 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4151 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4152 }
4153 Assert(!fInterceptMovDRx);
4154 }
4155 /*
4156 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4157 * must intercept #DB in order to maintain a correct DR6 guest value, and
4158 * because we need to intercept it to prevent nested #DBs from hanging the
4159 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4160 */
4161#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4162 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4163 && !CPUMIsGuestDebugStateActive(pVCpu))
4164#else
4165 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4166#endif
4167 {
4168 fInterceptMovDRx = true;
4169 }
4170
4171 /* Update DR7 with the actual guest value. */
4172 u32GuestDr7 = pMixedCtx->dr[7];
4173 pVCpu->hm.s.fUsingHyperDR7 = false;
4174 }
4175
4176 if (fInterceptMovDRx)
4177 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4178 else
4179 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4180
4181 /*
4182 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4183 * monitor-trap flag and update our cache.
4184 */
4185 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4186 {
4187 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4188 AssertRCReturn(rc2, rc2);
4189 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4190 }
4191
4192 /*
4193 * Update guest DR7.
4194 */
4195 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4196 AssertRCReturn(rc, rc);
4197
4198 return VINF_SUCCESS;
4199}
4200
4201
4202#ifdef VBOX_STRICT
4203/**
4204 * Strict function to validate segment registers.
4205 *
4206 * @param pVCpu The cross context virtual CPU structure.
4207 * @param pCtx Pointer to the guest-CPU context.
4208 *
4209 * @remarks Will import guest CR0 on strict builds during validation of
4210 * segments.
4211 */
4212static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4213{
4214 /*
4215 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4216 *
4217 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4218 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4219 * and doesn't change the guest-context value.
4220 */
4221 PVM pVM = pVCpu->CTX_SUFF(pVM);
4222 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4223 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4224 && ( !CPUMIsGuestInRealModeEx(pCtx)
4225 && !CPUMIsGuestInV86ModeEx(pCtx)))
4226 {
4227 /* Protected mode checks */
4228 /* CS */
4229 Assert(pCtx->cs.Attr.n.u1Present);
4230 Assert(!(pCtx->cs.Attr.u & 0xf00));
4231 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4232 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4233 || !(pCtx->cs.Attr.n.u1Granularity));
4234 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4235 || (pCtx->cs.Attr.n.u1Granularity));
4236 /* CS cannot be loaded with NULL in protected mode. */
4237 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4238 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4239 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4240 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4241 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4242 else
4243 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4244 /* SS */
4245 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4246 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4247 if ( !(pCtx->cr0 & X86_CR0_PE)
4248 || pCtx->cs.Attr.n.u4Type == 3)
4249 {
4250 Assert(!pCtx->ss.Attr.n.u2Dpl);
4251 }
4252 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4253 {
4254 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4255 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4256 Assert(pCtx->ss.Attr.n.u1Present);
4257 Assert(!(pCtx->ss.Attr.u & 0xf00));
4258 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4259 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4260 || !(pCtx->ss.Attr.n.u1Granularity));
4261 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4262 || (pCtx->ss.Attr.n.u1Granularity));
4263 }
4264 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4265 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4266 {
4267 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4268 Assert(pCtx->ds.Attr.n.u1Present);
4269 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4270 Assert(!(pCtx->ds.Attr.u & 0xf00));
4271 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4272 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4273 || !(pCtx->ds.Attr.n.u1Granularity));
4274 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4275 || (pCtx->ds.Attr.n.u1Granularity));
4276 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4277 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4278 }
4279 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4280 {
4281 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4282 Assert(pCtx->es.Attr.n.u1Present);
4283 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4284 Assert(!(pCtx->es.Attr.u & 0xf00));
4285 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4286 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4287 || !(pCtx->es.Attr.n.u1Granularity));
4288 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4289 || (pCtx->es.Attr.n.u1Granularity));
4290 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4291 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4292 }
4293 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->fs.Attr.n.u1Present);
4297 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->fs.Attr.u & 0xf00));
4299 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->fs.Attr.n.u1Granularity));
4302 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4303 || (pCtx->fs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->gs.Attr.n.u1Present);
4311 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->gs.Attr.u & 0xf00));
4313 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->gs.Attr.n.u1Granularity));
4316 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4317 || (pCtx->gs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 /* 64-bit capable CPUs. */
4322# if HC_ARCH_BITS == 64
4323 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4324 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4325 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4326 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4327# endif
4328 }
4329 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4330 || ( CPUMIsGuestInRealModeEx(pCtx)
4331 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4332 {
4333 /* Real and v86 mode checks. */
4334 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4335 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4336 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4337 {
4338 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4339 }
4340 else
4341 {
4342 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4343 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4344 }
4345
4346 /* CS */
4347 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4348 Assert(pCtx->cs.u32Limit == 0xffff);
4349 Assert(u32CSAttr == 0xf3);
4350 /* SS */
4351 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4352 Assert(pCtx->ss.u32Limit == 0xffff);
4353 Assert(u32SSAttr == 0xf3);
4354 /* DS */
4355 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4356 Assert(pCtx->ds.u32Limit == 0xffff);
4357 Assert(u32DSAttr == 0xf3);
4358 /* ES */
4359 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4360 Assert(pCtx->es.u32Limit == 0xffff);
4361 Assert(u32ESAttr == 0xf3);
4362 /* FS */
4363 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4364 Assert(pCtx->fs.u32Limit == 0xffff);
4365 Assert(u32FSAttr == 0xf3);
4366 /* GS */
4367 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4368 Assert(pCtx->gs.u32Limit == 0xffff);
4369 Assert(u32GSAttr == 0xf3);
4370 /* 64-bit capable CPUs. */
4371# if HC_ARCH_BITS == 64
4372 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4373 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4374 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4375 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4376# endif
4377 }
4378}
4379#endif /* VBOX_STRICT */
4380
4381
4382/**
4383 * Writes a guest segment register into the guest-state area in the VMCS.
4384 *
4385 * @returns VBox status code.
4386 * @param pVCpu The cross context virtual CPU structure.
4387 * @param idxSel Index of the selector in the VMCS.
4388 * @param idxLimit Index of the segment limit in the VMCS.
4389 * @param idxBase Index of the segment base in the VMCS.
4390 * @param idxAccess Index of the access rights of the segment in the VMCS.
4391 * @param pSelReg Pointer to the segment selector.
4392 *
4393 * @remarks No-long-jump zone!!!
4394 */
4395static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4396 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4397{
4398 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4399 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4400 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4401 AssertRCReturn(rc, rc);
4402
4403 uint32_t u32Access = pSelReg->Attr.u;
4404 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4405 {
4406 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4407 u32Access = 0xf3;
4408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4409 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4410 }
4411 else
4412 {
4413 /*
4414 * The way to differentiate between whether this is really a null selector or was just
4415 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4416 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4417 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4418 * NULL selectors loaded in protected-mode have their attribute as 0.
4419 */
4420 if (!u32Access)
4421 u32Access = X86DESCATTR_UNUSABLE;
4422 }
4423
4424 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4425 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4426 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4427
4428 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4429 AssertRCReturn(rc, rc);
4430 return rc;
4431}
4432
4433
4434/**
4435 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4436 * into the guest-state area in the VMCS.
4437 *
4438 * @returns VBox status code.
4439 * @param pVCpu The cross context virtual CPU structure.
4440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4441 * out-of-sync. Make sure to update the required fields
4442 * before using them.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452
4453 /*
4454 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4455 */
4456 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4457 {
4458 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4459 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4460 {
4461 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4462 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4463 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4464 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4465 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4466 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4467 }
4468
4469#ifdef VBOX_WITH_REM
4470 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4471 {
4472 Assert(pVM->hm.s.vmx.pRealModeTSS);
4473 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4474 if ( pVCpu->hm.s.vmx.fWasInRealMode
4475 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4476 {
4477 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4478 in real-mode (e.g. OpenBSD 4.0) */
4479 REMFlushTBs(pVM);
4480 Log4Func(("Switch to protected mode detected!\n"));
4481 pVCpu->hm.s.vmx.fWasInRealMode = false;
4482 }
4483 }
4484#endif
4485 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4486 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4487 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4488 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4489 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4490 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4491 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4492 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4493 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4494 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4495 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4496 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4497 AssertRCReturn(rc, rc);
4498
4499#ifdef VBOX_STRICT
4500 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4501#endif
4502
4503 /* Update the exit history entry with the correct CS.BASE + RIP. */
4504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4505 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4506
4507 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4508 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4509 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4510 }
4511
4512 /*
4513 * Guest TR.
4514 */
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4516 {
4517 /*
4518 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4519 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4520 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4521 */
4522 uint16_t u16Sel = 0;
4523 uint32_t u32Limit = 0;
4524 uint64_t u64Base = 0;
4525 uint32_t u32AccessRights = 0;
4526
4527 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4528 {
4529 u16Sel = pMixedCtx->tr.Sel;
4530 u32Limit = pMixedCtx->tr.u32Limit;
4531 u64Base = pMixedCtx->tr.u64Base;
4532 u32AccessRights = pMixedCtx->tr.Attr.u;
4533 }
4534 else
4535 {
4536 Assert(pVM->hm.s.vmx.pRealModeTSS);
4537 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4538
4539 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4540 RTGCPHYS GCPhys;
4541 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4542 AssertRCReturn(rc, rc);
4543
4544 X86DESCATTR DescAttr;
4545 DescAttr.u = 0;
4546 DescAttr.n.u1Present = 1;
4547 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4548
4549 u16Sel = 0;
4550 u32Limit = HM_VTX_TSS_SIZE;
4551 u64Base = GCPhys; /* in real-mode phys = virt. */
4552 u32AccessRights = DescAttr.u;
4553 }
4554
4555 /* Validate. */
4556 Assert(!(u16Sel & RT_BIT(2)));
4557 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4558 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4559 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4560 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4561 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4562 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4563 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4564 Assert( (u32Limit & 0xfff) == 0xfff
4565 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4566 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4567 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4568
4569 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4570 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4571 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4572 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4573 AssertRCReturn(rc, rc);
4574
4575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4576 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4577 }
4578
4579 /*
4580 * Guest GDTR.
4581 */
4582 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4583 {
4584 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4585 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4586 AssertRCReturn(rc, rc);
4587
4588 /* Validate. */
4589 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4590
4591 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4592 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4593 }
4594
4595 /*
4596 * Guest LDTR.
4597 */
4598 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4599 {
4600 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4601 uint32_t u32Access = 0;
4602 if (!pMixedCtx->ldtr.Attr.u)
4603 u32Access = X86DESCATTR_UNUSABLE;
4604 else
4605 u32Access = pMixedCtx->ldtr.Attr.u;
4606
4607 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4609 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4610 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4611 AssertRCReturn(rc, rc);
4612
4613 /* Validate. */
4614 if (!(u32Access & X86DESCATTR_UNUSABLE))
4615 {
4616 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4617 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4618 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4619 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4620 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4621 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4622 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4623 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4624 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4625 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4626 }
4627
4628 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4629 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4630 }
4631
4632 /*
4633 * Guest IDTR.
4634 */
4635 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4636 {
4637 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4638 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4639 AssertRCReturn(rc, rc);
4640
4641 /* Validate. */
4642 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4643
4644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4645 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4646 }
4647
4648 return VINF_SUCCESS;
4649}
4650
4651
4652/**
4653 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4654 * areas.
4655 *
4656 * These MSRs will automatically be loaded to the host CPU on every successful
4657 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4658 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4659 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4660 *
4661 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4662 *
4663 * @returns VBox status code.
4664 * @param pVCpu The cross context virtual CPU structure.
4665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4666 * out-of-sync. Make sure to update the required fields
4667 * before using them.
4668 *
4669 * @remarks No-long-jump zone!!!
4670 */
4671static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4672{
4673 AssertPtr(pVCpu);
4674 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4675
4676 /*
4677 * MSRs that we use the auto-load/store MSR area in the VMCS.
4678 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4679 */
4680 PVM pVM = pVCpu->CTX_SUFF(pVM);
4681 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4682 {
4683 if (pVM->hm.s.fAllow64BitGuests)
4684 {
4685#if HC_ARCH_BITS == 32
4686 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4687 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4688 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4689 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4690 AssertRCReturn(rc, rc);
4691# ifdef LOG_ENABLED
4692 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4693 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4694 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4695# endif
4696#endif
4697 }
4698 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4699 }
4700
4701 /*
4702 * Guest Sysenter MSRs.
4703 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4704 * VM-exits on WRMSRs for these MSRs.
4705 */
4706 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4707 {
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4709 {
4710 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4711 AssertRCReturn(rc, rc);
4712 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4713 }
4714
4715 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4716 {
4717 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4718 AssertRCReturn(rc, rc);
4719 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4720 }
4721
4722 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4723 {
4724 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4725 AssertRCReturn(rc, rc);
4726 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4727 }
4728 }
4729
4730 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4731 {
4732 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4733 {
4734 /*
4735 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4736 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4737 */
4738 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4739 {
4740 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4741 AssertRCReturn(rc,rc);
4742 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4743 }
4744 else
4745 {
4746 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4747 NULL /* pfAddedAndUpdated */);
4748 AssertRCReturn(rc, rc);
4749
4750 /* We need to intercept reads too, see @bugref{7386#c16}. */
4751 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4752 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4753 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4754 pVCpu->hm.s.vmx.cMsrs));
4755 }
4756 }
4757 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4758 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4760 }
4761
4762 return VINF_SUCCESS;
4763}
4764
4765
4766#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4767/**
4768 * Check if guest state allows safe use of 32-bit switcher again.
4769 *
4770 * Segment bases and protected mode structures must be 32-bit addressable
4771 * because the 32-bit switcher will ignore high dword when writing these VMCS
4772 * fields. See @bugref{8432} for details.
4773 *
4774 * @returns true if safe, false if must continue to use the 64-bit switcher.
4775 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4776 * out-of-sync. Make sure to update the required fields
4777 * before using them.
4778 *
4779 * @remarks No-long-jump zone!!!
4780 */
4781static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4782{
4783 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4784 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4785 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4786 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4787 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4788 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4789 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4790 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4791 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4792 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4793
4794 /* All good, bases are 32-bit. */
4795 return true;
4796}
4797#endif
4798
4799
4800/**
4801 * Selects up the appropriate function to run guest code.
4802 *
4803 * @returns VBox status code.
4804 * @param pVCpu The cross context virtual CPU structure.
4805 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4806 * out-of-sync. Make sure to update the required fields
4807 * before using them.
4808 *
4809 * @remarks No-long-jump zone!!!
4810 */
4811static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4812{
4813 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4814 {
4815#ifndef VBOX_ENABLE_64_BITS_GUESTS
4816 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4817#endif
4818 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4819#if HC_ARCH_BITS == 32
4820 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4821 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4822 {
4823#ifdef VBOX_STRICT
4824 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4825 {
4826 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4827 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4828 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4829 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4830 | HM_CHANGED_VMX_ENTRY_CTLS
4831 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4832 }
4833#endif
4834 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4835
4836 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4837 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4838 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4839 Log4Func(("Selected 64-bit switcher\n"));
4840 }
4841#else
4842 /* 64-bit host. */
4843 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4844#endif
4845 }
4846 else
4847 {
4848 /* Guest is not in long mode, use the 32-bit handler. */
4849#if HC_ARCH_BITS == 32
4850 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4851 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4852 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4853 {
4854# ifdef VBOX_STRICT
4855 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4856 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4857 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4858 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4859 | HM_CHANGED_VMX_ENTRY_CTLS
4860 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4861# endif
4862 }
4863# ifdef VBOX_ENABLE_64_BITS_GUESTS
4864 /*
4865 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4866 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4867 * switcher flag because now we know the guest is in a sane state where it's safe
4868 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4869 * the much faster 32-bit switcher again.
4870 */
4871 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4872 {
4873 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4874 Log4Func(("Selected 32-bit switcher\n"));
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4876 }
4877 else
4878 {
4879 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4880 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4881 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4882 {
4883 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4884 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4885 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4886 | HM_CHANGED_VMX_ENTRY_CTLS
4887 | HM_CHANGED_VMX_EXIT_CTLS
4888 | HM_CHANGED_HOST_CONTEXT);
4889 Log4Func(("Selected 32-bit switcher (safe)\n"));
4890 }
4891 }
4892# else
4893 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4894# endif
4895#else
4896 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4897#endif
4898 }
4899 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4900 return VINF_SUCCESS;
4901}
4902
4903
4904/**
4905 * Wrapper for running the guest code in VT-x.
4906 *
4907 * @returns VBox status code, no informational status codes.
4908 * @param pVCpu The cross context virtual CPU structure.
4909 * @param pCtx Pointer to the guest-CPU context.
4910 *
4911 * @remarks No-long-jump zone!!!
4912 */
4913DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4914{
4915 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4916 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4917
4918 /*
4919 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4920 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4921 * callee-saved and thus the need for this XMM wrapper.
4922 *
4923 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4924 */
4925 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4926 /** @todo Add stats for resume vs launch. */
4927 PVM pVM = pVCpu->CTX_SUFF(pVM);
4928#ifdef VBOX_WITH_KERNEL_USING_XMM
4929 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4930#else
4931 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4932#endif
4933 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4934 return rc;
4935}
4936
4937
4938/**
4939 * Reports world-switch error and dumps some useful debug info.
4940 *
4941 * @param pVCpu The cross context virtual CPU structure.
4942 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4943 * @param pCtx Pointer to the guest-CPU context.
4944 * @param pVmxTransient Pointer to the VMX transient structure (only
4945 * exitReason updated).
4946 */
4947static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4948{
4949 Assert(pVCpu);
4950 Assert(pCtx);
4951 Assert(pVmxTransient);
4952 HMVMX_ASSERT_PREEMPT_SAFE();
4953
4954 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4955 switch (rcVMRun)
4956 {
4957 case VERR_VMX_INVALID_VMXON_PTR:
4958 AssertFailed();
4959 break;
4960 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4961 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4962 {
4963 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4964 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4965 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4966 AssertRC(rc);
4967
4968 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4969 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4970 Cannot do it here as we may have been long preempted. */
4971
4972#ifdef VBOX_STRICT
4973 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4974 pVmxTransient->uExitReason));
4975 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4976 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4977 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4978 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4979 else
4980 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4981 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4982 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4983
4984 /* VMX control bits. */
4985 uint32_t u32Val;
4986 uint64_t u64Val;
4987 RTHCUINTREG uHCReg;
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4990 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4991 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4992 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4993 {
4994 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4995 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4996 }
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5009 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5010 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5012 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5013 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5014 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5016 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5018 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5020 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5021 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5022 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5024 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5025 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5026 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5027 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5028 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5029 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5030 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5031 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5032 {
5033 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5034 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5035 }
5036
5037 /* Guest bits. */
5038 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5039 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5040 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5041 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5042 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5043 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5044 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5045 {
5046 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5048 }
5049
5050 /* Host bits. */
5051 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5052 Log4(("Host CR0 %#RHr\n", uHCReg));
5053 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5054 Log4(("Host CR3 %#RHr\n", uHCReg));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5056 Log4(("Host CR4 %#RHr\n", uHCReg));
5057
5058 RTGDTR HostGdtr;
5059 PCX86DESCHC pDesc;
5060 ASMGetGDTR(&HostGdtr);
5061 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5062 Log4(("Host CS %#08x\n", u32Val));
5063 if (u32Val < HostGdtr.cbGdt)
5064 {
5065 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5066 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5067 }
5068
5069 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5070 Log4(("Host DS %#08x\n", u32Val));
5071 if (u32Val < HostGdtr.cbGdt)
5072 {
5073 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5074 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5075 }
5076
5077 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5078 Log4(("Host ES %#08x\n", u32Val));
5079 if (u32Val < HostGdtr.cbGdt)
5080 {
5081 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5082 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5083 }
5084
5085 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5086 Log4(("Host FS %#08x\n", u32Val));
5087 if (u32Val < HostGdtr.cbGdt)
5088 {
5089 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5090 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5091 }
5092
5093 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5094 Log4(("Host GS %#08x\n", u32Val));
5095 if (u32Val < HostGdtr.cbGdt)
5096 {
5097 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5098 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5099 }
5100
5101 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5102 Log4(("Host SS %#08x\n", u32Val));
5103 if (u32Val < HostGdtr.cbGdt)
5104 {
5105 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5106 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5107 }
5108
5109 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5110 Log4(("Host TR %#08x\n", u32Val));
5111 if (u32Val < HostGdtr.cbGdt)
5112 {
5113 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5114 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5115 }
5116
5117 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5118 Log4(("Host TR Base %#RHv\n", uHCReg));
5119 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5120 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5121 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5122 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5123 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5124 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5125 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5126 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5127 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5128 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5129 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5130 Log4(("Host RSP %#RHv\n", uHCReg));
5131 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5132 Log4(("Host RIP %#RHv\n", uHCReg));
5133# if HC_ARCH_BITS == 64
5134 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5135 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5136 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5137 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5138 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5139 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5140# endif
5141#endif /* VBOX_STRICT */
5142 break;
5143 }
5144
5145 default:
5146 /* Impossible */
5147 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5148 break;
5149 }
5150 NOREF(pCtx);
5151}
5152
5153
5154#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5155#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5156# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5157#endif
5158#ifdef VBOX_STRICT
5159static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5160{
5161 switch (idxField)
5162 {
5163 case VMX_VMCS_GUEST_RIP:
5164 case VMX_VMCS_GUEST_RSP:
5165 case VMX_VMCS_GUEST_SYSENTER_EIP:
5166 case VMX_VMCS_GUEST_SYSENTER_ESP:
5167 case VMX_VMCS_GUEST_GDTR_BASE:
5168 case VMX_VMCS_GUEST_IDTR_BASE:
5169 case VMX_VMCS_GUEST_CS_BASE:
5170 case VMX_VMCS_GUEST_DS_BASE:
5171 case VMX_VMCS_GUEST_ES_BASE:
5172 case VMX_VMCS_GUEST_FS_BASE:
5173 case VMX_VMCS_GUEST_GS_BASE:
5174 case VMX_VMCS_GUEST_SS_BASE:
5175 case VMX_VMCS_GUEST_LDTR_BASE:
5176 case VMX_VMCS_GUEST_TR_BASE:
5177 case VMX_VMCS_GUEST_CR3:
5178 return true;
5179 }
5180 return false;
5181}
5182
5183static bool hmR0VmxIsValidReadField(uint32_t idxField)
5184{
5185 switch (idxField)
5186 {
5187 /* Read-only fields. */
5188 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5189 return true;
5190 }
5191 /* Remaining readable fields should also be writable. */
5192 return hmR0VmxIsValidWriteField(idxField);
5193}
5194#endif /* VBOX_STRICT */
5195
5196
5197/**
5198 * Executes the specified handler in 64-bit mode.
5199 *
5200 * @returns VBox status code (no informational status codes).
5201 * @param pVCpu The cross context virtual CPU structure.
5202 * @param enmOp The operation to perform.
5203 * @param cParams Number of parameters.
5204 * @param paParam Array of 32-bit parameters.
5205 */
5206VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5207{
5208 PVM pVM = pVCpu->CTX_SUFF(pVM);
5209 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5210 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5211 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5212 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5213
5214#ifdef VBOX_STRICT
5215 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5216 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5217
5218 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5219 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5220#endif
5221
5222 /* Disable interrupts. */
5223 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5224
5225#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5226 RTCPUID idHostCpu = RTMpCpuId();
5227 CPUMR0SetLApic(pVCpu, idHostCpu);
5228#endif
5229
5230 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5231 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5232
5233 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5234 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5235 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5236
5237 /* Leave VMX Root Mode. */
5238 VMXDisable();
5239
5240 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5241
5242 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5243 CPUMSetHyperEIP(pVCpu, enmOp);
5244 for (int i = (int)cParams - 1; i >= 0; i--)
5245 CPUMPushHyper(pVCpu, paParam[i]);
5246
5247 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5248
5249 /* Call the switcher. */
5250 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5251 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5252
5253 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5254 /* Make sure the VMX instructions don't cause #UD faults. */
5255 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5256
5257 /* Re-enter VMX Root Mode */
5258 int rc2 = VMXEnable(HCPhysCpuPage);
5259 if (RT_FAILURE(rc2))
5260 {
5261 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5262 ASMSetFlags(fOldEFlags);
5263 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5264 return rc2;
5265 }
5266
5267 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5268 AssertRC(rc2);
5269 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5270 Assert(!(ASMGetFlags() & X86_EFL_IF));
5271 ASMSetFlags(fOldEFlags);
5272 return rc;
5273}
5274
5275
5276/**
5277 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5278 * supporting 64-bit guests.
5279 *
5280 * @returns VBox status code.
5281 * @param fResume Whether to VMLAUNCH or VMRESUME.
5282 * @param pCtx Pointer to the guest-CPU context.
5283 * @param pCache Pointer to the VMCS cache.
5284 * @param pVM The cross context VM structure.
5285 * @param pVCpu The cross context virtual CPU structure.
5286 */
5287DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5288{
5289 NOREF(fResume);
5290
5291 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5292 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5293
5294#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5295 pCache->uPos = 1;
5296 pCache->interPD = PGMGetInterPaeCR3(pVM);
5297 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5298#endif
5299
5300#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5301 pCache->TestIn.HCPhysCpuPage = 0;
5302 pCache->TestIn.HCPhysVmcs = 0;
5303 pCache->TestIn.pCache = 0;
5304 pCache->TestOut.HCPhysVmcs = 0;
5305 pCache->TestOut.pCache = 0;
5306 pCache->TestOut.pCtx = 0;
5307 pCache->TestOut.eflags = 0;
5308#else
5309 NOREF(pCache);
5310#endif
5311
5312 uint32_t aParam[10];
5313 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5314 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5315 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5316 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5317 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5318 aParam[5] = 0;
5319 aParam[6] = VM_RC_ADDR(pVM, pVM);
5320 aParam[7] = 0;
5321 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5322 aParam[9] = 0;
5323
5324#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5325 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5326 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5327#endif
5328 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5329
5330#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5331 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5332 Assert(pCtx->dr[4] == 10);
5333 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5334#endif
5335
5336#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5337 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5338 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5339 pVCpu->hm.s.vmx.HCPhysVmcs));
5340 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5341 pCache->TestOut.HCPhysVmcs));
5342 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5343 pCache->TestOut.pCache));
5344 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5345 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5346 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5347 pCache->TestOut.pCtx));
5348 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5349#endif
5350 NOREF(pCtx);
5351 return rc;
5352}
5353
5354
5355/**
5356 * Initialize the VMCS-Read cache.
5357 *
5358 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5359 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5360 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5361 * (those that have a 32-bit FULL & HIGH part).
5362 *
5363 * @returns VBox status code.
5364 * @param pVCpu The cross context virtual CPU structure.
5365 */
5366static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5367{
5368#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5369 do { \
5370 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5371 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5372 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5373 ++cReadFields; \
5374 } while (0)
5375
5376 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5377 uint32_t cReadFields = 0;
5378
5379 /*
5380 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5381 * and serve to indicate exceptions to the rules.
5382 */
5383
5384 /* Guest-natural selector base fields. */
5385#if 0
5386 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5387 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5389#endif
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5400 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5401 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5402#if 0
5403 /* Unused natural width guest-state fields. */
5404 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5406#endif
5407 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5408 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5409
5410 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5411 these 64-bit fields (using "FULL" and "HIGH" fields). */
5412#if 0
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5421 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5422#endif
5423
5424 /* Natural width guest-state fields. */
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5426#if 0
5427 /* Currently unused field. */
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5429#endif
5430
5431 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5432 {
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5434 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5435 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5436 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5437 }
5438 else
5439 {
5440 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5441 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5442 }
5443
5444#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5445 return VINF_SUCCESS;
5446}
5447
5448
5449/**
5450 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5451 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5452 * darwin, running 64-bit guests).
5453 *
5454 * @returns VBox status code.
5455 * @param pVCpu The cross context virtual CPU structure.
5456 * @param idxField The VMCS field encoding.
5457 * @param u64Val 16, 32 or 64-bit value.
5458 */
5459VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5460{
5461 int rc;
5462 switch (idxField)
5463 {
5464 /*
5465 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5466 */
5467 /* 64-bit Control fields. */
5468 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5469 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5470 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5471 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5472 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5473 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5474 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5475 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5476 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5477 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5478 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5479 case VMX_VMCS64_CTRL_EPTP_FULL:
5480 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5481 /* 64-bit Guest-state fields. */
5482 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5483 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5484 case VMX_VMCS64_GUEST_PAT_FULL:
5485 case VMX_VMCS64_GUEST_EFER_FULL:
5486 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5487 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5488 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5489 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5490 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5491 /* 64-bit Host-state fields. */
5492 case VMX_VMCS64_HOST_PAT_FULL:
5493 case VMX_VMCS64_HOST_EFER_FULL:
5494 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5495 {
5496 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5497 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5498 break;
5499 }
5500
5501 /*
5502 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5503 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5504 */
5505 /* Natural-width Guest-state fields. */
5506 case VMX_VMCS_GUEST_CR3:
5507 case VMX_VMCS_GUEST_ES_BASE:
5508 case VMX_VMCS_GUEST_CS_BASE:
5509 case VMX_VMCS_GUEST_SS_BASE:
5510 case VMX_VMCS_GUEST_DS_BASE:
5511 case VMX_VMCS_GUEST_FS_BASE:
5512 case VMX_VMCS_GUEST_GS_BASE:
5513 case VMX_VMCS_GUEST_LDTR_BASE:
5514 case VMX_VMCS_GUEST_TR_BASE:
5515 case VMX_VMCS_GUEST_GDTR_BASE:
5516 case VMX_VMCS_GUEST_IDTR_BASE:
5517 case VMX_VMCS_GUEST_RSP:
5518 case VMX_VMCS_GUEST_RIP:
5519 case VMX_VMCS_GUEST_SYSENTER_ESP:
5520 case VMX_VMCS_GUEST_SYSENTER_EIP:
5521 {
5522 if (!(RT_HI_U32(u64Val)))
5523 {
5524 /* If this field is 64-bit, VT-x will zero out the top bits. */
5525 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5526 }
5527 else
5528 {
5529 /* Assert that only the 32->64 switcher case should ever come here. */
5530 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5531 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5532 }
5533 break;
5534 }
5535
5536 default:
5537 {
5538 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5539 rc = VERR_INVALID_PARAMETER;
5540 break;
5541 }
5542 }
5543 AssertRCReturn(rc, rc);
5544 return rc;
5545}
5546
5547
5548/**
5549 * Queue up a VMWRITE by using the VMCS write cache.
5550 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5551 *
5552 * @param pVCpu The cross context virtual CPU structure.
5553 * @param idxField The VMCS field encoding.
5554 * @param u64Val 16, 32 or 64-bit value.
5555 */
5556VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5557{
5558 AssertPtr(pVCpu);
5559 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5560
5561 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5562 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5563
5564 /* Make sure there are no duplicates. */
5565 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5566 {
5567 if (pCache->Write.aField[i] == idxField)
5568 {
5569 pCache->Write.aFieldVal[i] = u64Val;
5570 return VINF_SUCCESS;
5571 }
5572 }
5573
5574 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5575 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5576 pCache->Write.cValidEntries++;
5577 return VINF_SUCCESS;
5578}
5579#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5580
5581
5582/**
5583 * Sets up the usage of TSC-offsetting and updates the VMCS.
5584 *
5585 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5586 * VMX preemption timer.
5587 *
5588 * @returns VBox status code.
5589 * @param pVCpu The cross context virtual CPU structure.
5590 *
5591 * @remarks No-long-jump zone!!!
5592 */
5593static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5594{
5595 bool fOffsettedTsc;
5596 bool fParavirtTsc;
5597 PVM pVM = pVCpu->CTX_SUFF(pVM);
5598 uint64_t uTscOffset;
5599 if (pVM->hm.s.vmx.fUsePreemptTimer)
5600 {
5601 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5602
5603 /* Make sure the returned values have sane upper and lower boundaries. */
5604 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5605 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5606 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5607 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5608
5609 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5610 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5611 AssertRC(rc);
5612 }
5613 else
5614 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5615
5616 /** @todo later optimize this to be done elsewhere and not before every
5617 * VM-entry. */
5618 if (fParavirtTsc)
5619 {
5620 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5621 information before every VM-entry, hence disable it for performance sake. */
5622#if 0
5623 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5624 AssertRC(rc);
5625#endif
5626 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5627 }
5628
5629 if ( fOffsettedTsc
5630 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5631 {
5632 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5633 {
5634 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TscOffset);
5635 AssertRC(rc);
5636 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5637 }
5638
5639 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5640 {
5641 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5642 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5643 AssertRC(rc);
5644 }
5645 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5646 }
5647 else
5648 {
5649 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5650 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5651 {
5652 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5653 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5654 AssertRC(rc);
5655 }
5656 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5657 }
5658}
5659
5660
5661/**
5662 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5663 * VM-exit interruption info type.
5664 *
5665 * @returns The IEM exception flags.
5666 * @param uVector The event vector.
5667 * @param uVmxVectorType The VMX event type.
5668 *
5669 * @remarks This function currently only constructs flags required for
5670 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5671 * and CR2 aspects of an exception are not included).
5672 */
5673static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5674{
5675 uint32_t fIemXcptFlags;
5676 switch (uVmxVectorType)
5677 {
5678 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5679 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5680 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5681 break;
5682
5683 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5684 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5685 break;
5686
5687 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5688 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5689 break;
5690
5691 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5692 {
5693 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5694 if (uVector == X86_XCPT_BP)
5695 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5696 else if (uVector == X86_XCPT_OF)
5697 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5698 else
5699 {
5700 fIemXcptFlags = 0;
5701 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5702 }
5703 break;
5704 }
5705
5706 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5707 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5708 break;
5709
5710 default:
5711 fIemXcptFlags = 0;
5712 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5713 break;
5714 }
5715 return fIemXcptFlags;
5716}
5717
5718
5719/**
5720 * Sets an event as a pending event to be injected into the guest.
5721 *
5722 * @param pVCpu The cross context virtual CPU structure.
5723 * @param u32IntInfo The VM-entry interruption-information field.
5724 * @param cbInstr The VM-entry instruction length in bytes (for software
5725 * interrupts, exceptions and privileged software
5726 * exceptions).
5727 * @param u32ErrCode The VM-entry exception error code.
5728 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5729 * page-fault.
5730 *
5731 * @remarks Statistics counter assumes this is a guest event being injected or
5732 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5733 * always incremented.
5734 */
5735DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5736 RTGCUINTPTR GCPtrFaultAddress)
5737{
5738 Assert(!pVCpu->hm.s.Event.fPending);
5739 pVCpu->hm.s.Event.fPending = true;
5740 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5741 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5742 pVCpu->hm.s.Event.cbInstr = cbInstr;
5743 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5744}
5745
5746
5747/**
5748 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5749 *
5750 * @param pVCpu The cross context virtual CPU structure.
5751 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5752 * out-of-sync. Make sure to update the required fields
5753 * before using them.
5754 */
5755DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5756{
5757 NOREF(pMixedCtx);
5758 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5759 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5760 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5761 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5762}
5763
5764
5765/**
5766 * Handle a condition that occurred while delivering an event through the guest
5767 * IDT.
5768 *
5769 * @returns Strict VBox status code (i.e. informational status codes too).
5770 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5771 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5772 * to continue execution of the guest which will delivery the \#DF.
5773 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5774 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5775 *
5776 * @param pVCpu The cross context virtual CPU structure.
5777 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5778 * out-of-sync. Make sure to update the required fields
5779 * before using them.
5780 * @param pVmxTransient Pointer to the VMX transient structure.
5781 *
5782 * @remarks No-long-jump zone!!!
5783 */
5784static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5785{
5786 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5787
5788 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5789 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5790 AssertRCReturn(rc2, rc2);
5791
5792 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5793 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5794 {
5795 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5796 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5797
5798 /*
5799 * If the event was a software interrupt (generated with INT n) or a software exception
5800 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5801 * can handle the VM-exit and continue guest execution which will re-execute the
5802 * instruction rather than re-injecting the exception, as that can cause premature
5803 * trips to ring-3 before injection and involve TRPM which currently has no way of
5804 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5805 * the problem).
5806 */
5807 IEMXCPTRAISE enmRaise;
5808 IEMXCPTRAISEINFO fRaiseInfo;
5809 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5810 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5811 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5812 {
5813 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5814 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5815 }
5816 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5817 {
5818 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5819 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5820 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5821 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5822 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5823 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5824 uExitVectorType), VERR_VMX_IPE_5);
5825
5826 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5827
5828 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5829 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5830 {
5831 pVmxTransient->fVectoringPF = true;
5832 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5833 }
5834 }
5835 else
5836 {
5837 /*
5838 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5839 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5840 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5841 */
5842 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5843 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5844 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5845 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5846 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5847 }
5848
5849 /*
5850 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5851 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5852 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5853 * subsequent VM-entry would fail.
5854 *
5855 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5856 */
5857 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5858 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5859 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5860 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5861 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5862 {
5863 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5864 }
5865
5866 switch (enmRaise)
5867 {
5868 case IEMXCPTRAISE_CURRENT_XCPT:
5869 {
5870 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5871 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5872 Assert(rcStrict == VINF_SUCCESS);
5873 break;
5874 }
5875
5876 case IEMXCPTRAISE_PREV_EVENT:
5877 {
5878 uint32_t u32ErrCode;
5879 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5880 {
5881 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5882 AssertRCReturn(rc2, rc2);
5883 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5884 }
5885 else
5886 u32ErrCode = 0;
5887
5888 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5889 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5890 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5891 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5892
5893 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5894 pVCpu->hm.s.Event.u32ErrCode));
5895 Assert(rcStrict == VINF_SUCCESS);
5896 break;
5897 }
5898
5899 case IEMXCPTRAISE_REEXEC_INSTR:
5900 Assert(rcStrict == VINF_SUCCESS);
5901 break;
5902
5903 case IEMXCPTRAISE_DOUBLE_FAULT:
5904 {
5905 /*
5906 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5907 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5908 */
5909 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5910 {
5911 pVmxTransient->fVectoringDoublePF = true;
5912 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5913 pMixedCtx->cr2));
5914 rcStrict = VINF_SUCCESS;
5915 }
5916 else
5917 {
5918 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5919 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5920 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5921 uIdtVector, uExitVector));
5922 rcStrict = VINF_HM_DOUBLE_FAULT;
5923 }
5924 break;
5925 }
5926
5927 case IEMXCPTRAISE_TRIPLE_FAULT:
5928 {
5929 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5930 rcStrict = VINF_EM_RESET;
5931 break;
5932 }
5933
5934 case IEMXCPTRAISE_CPU_HANG:
5935 {
5936 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5937 rcStrict = VERR_EM_GUEST_CPU_HANG;
5938 break;
5939 }
5940
5941 default:
5942 {
5943 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5944 rcStrict = VERR_VMX_IPE_2;
5945 break;
5946 }
5947 }
5948 }
5949 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5950 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5951 && uExitVector != X86_XCPT_DF
5952 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5953 {
5954 /*
5955 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5956 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5957 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5958 */
5959 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5960 {
5961 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5962 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5963 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5964 }
5965 }
5966
5967 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5968 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5969 return rcStrict;
5970}
5971
5972
5973/**
5974 * Imports a guest segment register from the current VMCS into
5975 * the guest-CPU context.
5976 *
5977 * @returns VBox status code.
5978 * @param pVCpu The cross context virtual CPU structure.
5979 * @param idxSel Index of the selector in the VMCS.
5980 * @param idxLimit Index of the segment limit in the VMCS.
5981 * @param idxBase Index of the segment base in the VMCS.
5982 * @param idxAccess Index of the access rights of the segment in the VMCS.
5983 * @param pSelReg Pointer to the segment selector.
5984 *
5985 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
5986 * do not log!
5987 *
5988 * @remarks Never call this function directly!!! Use the
5989 * HMVMX_IMPORT_SREG() macro as that takes care
5990 * of whether to read from the VMCS cache or not.
5991 */
5992static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5993 PCPUMSELREG pSelReg)
5994{
5995 NOREF(pVCpu);
5996
5997 uint32_t u32Sel;
5998 uint32_t u32Limit;
5999 uint32_t u32Attr;
6000 uint64_t u64Base;
6001 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6002 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6003 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6004 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6005 AssertRCReturn(rc, rc);
6006
6007 pSelReg->Sel = (uint16_t)u32Sel;
6008 pSelReg->ValidSel = (uint16_t)u32Sel;
6009 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6010 pSelReg->u32Limit = u32Limit;
6011 pSelReg->u64Base = u64Base;
6012 pSelReg->Attr.u = u32Attr;
6013
6014 /*
6015 * If VT-x marks the segment as unusable, most other bits remain undefined:
6016 * - For CS the L, D and G bits have meaning.
6017 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6018 * - For the remaining data segments no bits are defined.
6019 *
6020 * The present bit and the unusable bit has been observed to be set at the
6021 * same time (the selector was supposed to be invalid as we started executing
6022 * a V8086 interrupt in ring-0).
6023 *
6024 * What should be important for the rest of the VBox code, is that the P bit is
6025 * cleared. Some of the other VBox code recognizes the unusable bit, but
6026 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6027 * safe side here, we'll strip off P and other bits we don't care about. If
6028 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6029 *
6030 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6031 */
6032 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6033 {
6034 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6035
6036 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6037 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6038 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6039#ifdef VBOX_STRICT
6040 VMMRZCallRing3Disable(pVCpu);
6041 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6042# ifdef DEBUG_bird
6043 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6044 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6045 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6046# endif
6047 VMMRZCallRing3Enable(pVCpu);
6048#endif
6049 }
6050 return VINF_SUCCESS;
6051}
6052
6053
6054/**
6055 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6056 *
6057 * @returns VBox status code.
6058 * @param pVCpu The cross context virtual CPU structure.
6059 *
6060 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6061 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6062 * instead!!!
6063 */
6064DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6065{
6066 uint64_t u64Val;
6067 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6068 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6069 {
6070 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6071 if (RT_SUCCESS(rc))
6072 {
6073 pCtx->rip = u64Val;
6074 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6075 }
6076 return rc;
6077 }
6078 return VINF_SUCCESS;
6079}
6080
6081
6082/**
6083 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6084 *
6085 * @returns VBox status code.
6086 * @param pVCpu The cross context virtual CPU structure.
6087 *
6088 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6089 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6090 * instead!!!
6091 */
6092DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6093{
6094 uint32_t u32Val;
6095 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6096 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6097 {
6098 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6099 if (RT_SUCCESS(rc))
6100 {
6101 pCtx->eflags.u32 = u32Val;
6102
6103 /* Restore eflags for real-on-v86-mode hack. */
6104 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6105 {
6106 pCtx->eflags.Bits.u1VM = 0;
6107 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6108 }
6109 }
6110 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6111 return rc;
6112 }
6113 return VINF_SUCCESS;
6114}
6115
6116
6117/**
6118 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6119 * context.
6120 *
6121 * @returns VBox status code.
6122 * @param pVCpu The cross context virtual CPU structure.
6123 *
6124 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6125 * do not log!
6126 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6127 * instead!!!
6128 */
6129DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6130{
6131 uint32_t u32Val;
6132 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6133 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6134 if (RT_SUCCESS(rc))
6135 {
6136 /*
6137 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6138 * might need them in hmR0VmxEvaluatePendingEvent().
6139 */
6140 if (!u32Val)
6141 {
6142 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6143 {
6144 rc = hmR0VmxImportGuestRip(pVCpu);
6145 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6146 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6147 }
6148
6149 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6150 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6151 }
6152 else
6153 {
6154 rc = hmR0VmxImportGuestRip(pVCpu);
6155 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6156
6157 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6158 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6159 {
6160 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6161 }
6162 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6163 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6164
6165 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6166 {
6167 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6168 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6169 }
6170 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6171 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6172 }
6173 }
6174 return rc;
6175}
6176
6177
6178/**
6179 * Worker for VMXR0ImportStateOnDemand.
6180 *
6181 * @returns VBox status code.
6182 * @param pVCpu The cross context virtual CPU structure.
6183 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6184 */
6185static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6186{
6187#define VMXLOCAL_BREAK_RC(a_rc) \
6188 if (RT_FAILURE(a_rc)) \
6189 break
6190
6191 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6192
6193 int rc = VINF_SUCCESS;
6194 PVM pVM = pVCpu->CTX_SUFF(pVM);
6195 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6196 uint64_t u64Val;
6197 uint32_t u32Val;
6198
6199 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6200
6201 /*
6202 * We disable interrupts to make the updating of the state and in particular
6203 * the fExtrn modification atomic wrt to preemption hooks.
6204 */
6205 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6206
6207 fWhat &= pCtx->fExtrn;
6208 if (fWhat)
6209 {
6210 do
6211 {
6212 if (fWhat & CPUMCTX_EXTRN_RIP)
6213 {
6214 rc = hmR0VmxImportGuestRip(pVCpu);
6215 VMXLOCAL_BREAK_RC(rc);
6216 }
6217
6218 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6219 {
6220 rc = hmR0VmxImportGuestRFlags(pVCpu);
6221 VMXLOCAL_BREAK_RC(rc);
6222 }
6223
6224 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6225 {
6226 rc = hmR0VmxImportGuestIntrState(pVCpu);
6227 VMXLOCAL_BREAK_RC(rc);
6228 }
6229
6230 if (fWhat & CPUMCTX_EXTRN_RSP)
6231 {
6232 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6233 VMXLOCAL_BREAK_RC(rc);
6234 pCtx->rsp = u64Val;
6235 }
6236
6237 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6238 {
6239 if (fWhat & CPUMCTX_EXTRN_CS)
6240 {
6241 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6242 VMXLOCAL_BREAK_RC(rc);
6243 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6244 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6245 }
6246 if (fWhat & CPUMCTX_EXTRN_SS)
6247 {
6248 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6249 VMXLOCAL_BREAK_RC(rc);
6250 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6251 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6252 }
6253 if (fWhat & CPUMCTX_EXTRN_DS)
6254 {
6255 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6256 VMXLOCAL_BREAK_RC(rc);
6257 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6258 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6259 }
6260 if (fWhat & CPUMCTX_EXTRN_ES)
6261 {
6262 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6263 VMXLOCAL_BREAK_RC(rc);
6264 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6265 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6266 }
6267 if (fWhat & CPUMCTX_EXTRN_FS)
6268 {
6269 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6270 VMXLOCAL_BREAK_RC(rc);
6271 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6272 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6273 }
6274 if (fWhat & CPUMCTX_EXTRN_GS)
6275 {
6276 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6277 VMXLOCAL_BREAK_RC(rc);
6278 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6279 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6280 }
6281 }
6282
6283 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6284 {
6285 if (fWhat & CPUMCTX_EXTRN_LDTR)
6286 {
6287 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6288 VMXLOCAL_BREAK_RC(rc);
6289 }
6290
6291 if (fWhat & CPUMCTX_EXTRN_GDTR)
6292 {
6293 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6294 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6295 VMXLOCAL_BREAK_RC(rc);
6296 pCtx->gdtr.pGdt = u64Val;
6297 pCtx->gdtr.cbGdt = u32Val;
6298 }
6299
6300 /* Guest IDTR. */
6301 if (fWhat & CPUMCTX_EXTRN_IDTR)
6302 {
6303 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6304 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6305 VMXLOCAL_BREAK_RC(rc);
6306 pCtx->idtr.pIdt = u64Val;
6307 pCtx->idtr.cbIdt = u32Val;
6308 }
6309
6310 /* Guest TR. */
6311 if (fWhat & CPUMCTX_EXTRN_TR)
6312 {
6313 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6314 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6315 {
6316 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6317 VMXLOCAL_BREAK_RC(rc);
6318 }
6319 }
6320 }
6321
6322 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6323 {
6324 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6325 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6326 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6327 pCtx->SysEnter.cs = u32Val;
6328 VMXLOCAL_BREAK_RC(rc);
6329 }
6330
6331#if HC_ARCH_BITS == 64
6332 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6333 {
6334 if ( pVM->hm.s.fAllow64BitGuests
6335 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6336 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6337 }
6338
6339 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6340 {
6341 if ( pVM->hm.s.fAllow64BitGuests
6342 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6343 {
6344 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6345 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6346 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6347 }
6348 }
6349#endif
6350
6351 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6352#if HC_ARCH_BITS == 32
6353 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6354#endif
6355 )
6356 {
6357 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6358 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6359 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6360 {
6361 switch (pMsr->u32Msr)
6362 {
6363#if HC_ARCH_BITS == 32
6364 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6365 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6366 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6367 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6368#endif
6369 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6370 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6371 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6372 default:
6373 {
6374 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6375 ASMSetFlags(fEFlags);
6376 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6377 cMsrs));
6378 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6379 }
6380 }
6381 }
6382 }
6383
6384 if (fWhat & CPUMCTX_EXTRN_DR7)
6385 {
6386 if (!pVCpu->hm.s.fUsingHyperDR7)
6387 {
6388 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6389 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6390 VMXLOCAL_BREAK_RC(rc);
6391 pCtx->dr[7] = u32Val;
6392 }
6393 }
6394
6395 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6396 {
6397 uint32_t u32Shadow;
6398 if (fWhat & CPUMCTX_EXTRN_CR0)
6399 {
6400 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6401 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6402 VMXLOCAL_BREAK_RC(rc);
6403 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6404 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6405 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6406 CPUMSetGuestCR0(pVCpu, u32Val);
6407 VMMRZCallRing3Enable(pVCpu);
6408 }
6409
6410 if (fWhat & CPUMCTX_EXTRN_CR4)
6411 {
6412 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6413 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6414 VMXLOCAL_BREAK_RC(rc);
6415 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6416 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6417 CPUMSetGuestCR4(pVCpu, u32Val);
6418 }
6419
6420 if (fWhat & CPUMCTX_EXTRN_CR3)
6421 {
6422 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6423 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6424 || ( pVM->hm.s.fNestedPaging
6425 && CPUMIsGuestPagingEnabledEx(pCtx)))
6426 {
6427 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6428 if (pCtx->cr3 != u64Val)
6429 {
6430 CPUMSetGuestCR3(pVCpu, u64Val);
6431 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6432 }
6433
6434 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6435 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6436 if (CPUMIsGuestInPAEModeEx(pCtx))
6437 {
6438 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6439 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6440 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6441 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6442 VMXLOCAL_BREAK_RC(rc);
6443 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6444 }
6445 }
6446 }
6447 }
6448 } while (0);
6449
6450 if (RT_SUCCESS(rc))
6451 {
6452 /* Update fExtrn. */
6453 pCtx->fExtrn &= ~fWhat;
6454
6455 /* If everything has been imported, clear the HM keeper bit. */
6456 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6457 {
6458 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6459 Assert(!pCtx->fExtrn);
6460 }
6461 }
6462 }
6463 else
6464 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6465
6466 ASMSetFlags(fEFlags);
6467
6468 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6469
6470 /*
6471 * Honor any pending CR3 updates.
6472 *
6473 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6474 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6475 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6476 *
6477 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6478 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6479 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6480 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6481 *
6482 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6483 */
6484 if (VMMRZCallRing3IsEnabled(pVCpu))
6485 {
6486 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6487 {
6488 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6489 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6490 }
6491
6492 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6493 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6494
6495 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6496 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6497 }
6498
6499 return VINF_SUCCESS;
6500#undef VMXLOCAL_BREAK_RC
6501}
6502
6503
6504/**
6505 * Saves the guest state from the VMCS into the guest-CPU context.
6506 *
6507 * @returns VBox status code.
6508 * @param pVCpu The cross context virtual CPU structure.
6509 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6510 */
6511VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6512{
6513 return hmR0VmxImportGuestState(pVCpu, fWhat);
6514}
6515
6516
6517/**
6518 * Check per-VM and per-VCPU force flag actions that require us to go back to
6519 * ring-3 for one reason or another.
6520 *
6521 * @returns Strict VBox status code (i.e. informational status codes too)
6522 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6523 * ring-3.
6524 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6525 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6526 * interrupts)
6527 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6528 * all EMTs to be in ring-3.
6529 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6530 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6531 * to the EM loop.
6532 *
6533 * @param pVCpu The cross context virtual CPU structure.
6534 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6535 * out-of-sync. Make sure to update the required fields
6536 * before using them.
6537 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6538 */
6539static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6540{
6541 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6542
6543 /*
6544 * Anything pending? Should be more likely than not if we're doing a good job.
6545 */
6546 PVM pVM = pVCpu->CTX_SUFF(pVM);
6547 if ( !fStepping
6548 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6549 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6550 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6551 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6552 return VINF_SUCCESS;
6553
6554 /* Pending PGM C3 sync. */
6555 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6556 {
6557 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6558 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6559 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6560 if (rcStrict2 != VINF_SUCCESS)
6561 {
6562 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6563 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6564 return rcStrict2;
6565 }
6566 }
6567
6568 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6569 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6570 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6571 {
6572 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6573 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6574 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6575 return rc2;
6576 }
6577
6578 /* Pending VM request packets, such as hardware interrupts. */
6579 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6580 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6581 {
6582 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6583 return VINF_EM_PENDING_REQUEST;
6584 }
6585
6586 /* Pending PGM pool flushes. */
6587 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6588 {
6589 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6590 return VINF_PGM_POOL_FLUSH_PENDING;
6591 }
6592
6593 /* Pending DMA requests. */
6594 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6595 {
6596 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6597 return VINF_EM_RAW_TO_R3;
6598 }
6599
6600 return VINF_SUCCESS;
6601}
6602
6603
6604/**
6605 * Converts any TRPM trap into a pending HM event. This is typically used when
6606 * entering from ring-3 (not longjmp returns).
6607 *
6608 * @param pVCpu The cross context virtual CPU structure.
6609 */
6610static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6611{
6612 Assert(TRPMHasTrap(pVCpu));
6613 Assert(!pVCpu->hm.s.Event.fPending);
6614
6615 uint8_t uVector;
6616 TRPMEVENT enmTrpmEvent;
6617 RTGCUINT uErrCode;
6618 RTGCUINTPTR GCPtrFaultAddress;
6619 uint8_t cbInstr;
6620
6621 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6622 AssertRC(rc);
6623
6624 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6625 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6626 if (enmTrpmEvent == TRPM_TRAP)
6627 {
6628 switch (uVector)
6629 {
6630 case X86_XCPT_NMI:
6631 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6632 break;
6633
6634 case X86_XCPT_BP:
6635 case X86_XCPT_OF:
6636 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6637 break;
6638
6639 case X86_XCPT_PF:
6640 case X86_XCPT_DF:
6641 case X86_XCPT_TS:
6642 case X86_XCPT_NP:
6643 case X86_XCPT_SS:
6644 case X86_XCPT_GP:
6645 case X86_XCPT_AC:
6646 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6647 RT_FALL_THRU();
6648 default:
6649 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6650 break;
6651 }
6652 }
6653 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6654 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6655 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6656 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6657 else
6658 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6659
6660 rc = TRPMResetTrap(pVCpu);
6661 AssertRC(rc);
6662 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6663 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6664
6665 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6666}
6667
6668
6669/**
6670 * Converts the pending HM event into a TRPM trap.
6671 *
6672 * @param pVCpu The cross context virtual CPU structure.
6673 */
6674static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6675{
6676 Assert(pVCpu->hm.s.Event.fPending);
6677
6678 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6679 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6680 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6681 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6682
6683 /* If a trap was already pending, we did something wrong! */
6684 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6685
6686 TRPMEVENT enmTrapType;
6687 switch (uVectorType)
6688 {
6689 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6690 enmTrapType = TRPM_HARDWARE_INT;
6691 break;
6692
6693 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6694 enmTrapType = TRPM_SOFTWARE_INT;
6695 break;
6696
6697 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6698 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6699 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6700 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6701 enmTrapType = TRPM_TRAP;
6702 break;
6703
6704 default:
6705 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6706 enmTrapType = TRPM_32BIT_HACK;
6707 break;
6708 }
6709
6710 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6711
6712 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6713 AssertRC(rc);
6714
6715 if (fErrorCodeValid)
6716 TRPMSetErrorCode(pVCpu, uErrorCode);
6717
6718 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6719 && uVector == X86_XCPT_PF)
6720 {
6721 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6722 }
6723 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6724 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6725 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6726 {
6727 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6728 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6729 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6730 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6731 }
6732
6733 /* Clear any pending events from the VMCS. */
6734 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6735 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6736
6737 /* We're now done converting the pending event. */
6738 pVCpu->hm.s.Event.fPending = false;
6739}
6740
6741
6742/**
6743 * Does the necessary state syncing before returning to ring-3 for any reason
6744 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6745 *
6746 * @returns VBox status code.
6747 * @param pVCpu The cross context virtual CPU structure.
6748 * @param fImportState Whether to import the guest state from the VMCS back
6749 * to the guest-CPU context.
6750 *
6751 * @remarks No-long-jmp zone!!!
6752 */
6753static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6754{
6755 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6756 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6757
6758 RTCPUID idCpu = RTMpCpuId();
6759 Log4Func(("HostCpuId=%u\n", idCpu));
6760
6761 /*
6762 * !!! IMPORTANT !!!
6763 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6764 */
6765
6766 /* Save the guest state if necessary. */
6767 if (fImportState)
6768 {
6769 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6770 AssertRCReturn(rc, rc);
6771 }
6772
6773 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6774 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6775 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6776
6777 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6778#ifdef VBOX_STRICT
6779 if (CPUMIsHyperDebugStateActive(pVCpu))
6780 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6781#endif
6782 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6783 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6784 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6785
6786#if HC_ARCH_BITS == 64
6787 /* Restore host-state bits that VT-x only restores partially. */
6788 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6789 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6790 {
6791 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6792 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6793 }
6794 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6795#endif
6796
6797 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6798 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6799 {
6800 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6801 if (!fImportState)
6802 {
6803 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6804 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6805 AssertRCReturn(rc, rc);
6806 }
6807 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6808 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6809 }
6810 else
6811 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6812
6813 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6814 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6815
6816 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6817 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6818 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6819 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6820 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6821 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6822 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6823 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6824 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6825
6826 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6827
6828 /** @todo This partially defeats the purpose of having preemption hooks.
6829 * The problem is, deregistering the hooks should be moved to a place that
6830 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6831 * context.
6832 */
6833 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6834 {
6835 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6836 AssertRCReturn(rc, rc);
6837
6838 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6839 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6840 }
6841 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6842 NOREF(idCpu);
6843
6844 return VINF_SUCCESS;
6845}
6846
6847
6848/**
6849 * Leaves the VT-x session.
6850 *
6851 * @returns VBox status code.
6852 * @param pVCpu The cross context virtual CPU structure.
6853 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6854 * out-of-sync. Make sure to update the required fields
6855 * before using them.
6856 *
6857 * @remarks No-long-jmp zone!!!
6858 */
6859static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6860{
6861 HM_DISABLE_PREEMPT();
6862 HMVMX_ASSERT_CPU_SAFE();
6863 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6864 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6865
6866 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6867 and done this from the VMXR0ThreadCtxCallback(). */
6868 if (!pVCpu->hm.s.fLeaveDone)
6869 {
6870 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6871 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6872 pVCpu->hm.s.fLeaveDone = true;
6873 }
6874 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6875
6876 /*
6877 * !!! IMPORTANT !!!
6878 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6879 */
6880
6881 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6882 /** @todo Deregistering here means we need to VMCLEAR always
6883 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6884 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6885 VMMR0ThreadCtxHookDisable(pVCpu);
6886
6887 /* Leave HM context. This takes care of local init (term). */
6888 int rc = HMR0LeaveCpu(pVCpu);
6889
6890 HM_RESTORE_PREEMPT();
6891 return rc;
6892}
6893
6894
6895/**
6896 * Does the necessary state syncing before doing a longjmp to ring-3.
6897 *
6898 * @returns VBox status code.
6899 * @param pVCpu The cross context virtual CPU structure.
6900 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6901 * out-of-sync. Make sure to update the required fields
6902 * before using them.
6903 *
6904 * @remarks No-long-jmp zone!!!
6905 */
6906DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6907{
6908 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6909}
6910
6911
6912/**
6913 * Take necessary actions before going back to ring-3.
6914 *
6915 * An action requires us to go back to ring-3. This function does the necessary
6916 * steps before we can safely return to ring-3. This is not the same as longjmps
6917 * to ring-3, this is voluntary and prepares the guest so it may continue
6918 * executing outside HM (recompiler/IEM).
6919 *
6920 * @returns VBox status code.
6921 * @param pVCpu The cross context virtual CPU structure.
6922 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6923 * out-of-sync. Make sure to update the required fields
6924 * before using them.
6925 * @param rcExit The reason for exiting to ring-3. Can be
6926 * VINF_VMM_UNKNOWN_RING3_CALL.
6927 */
6928static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6929{
6930 Assert(pVCpu);
6931 Assert(pMixedCtx);
6932 HMVMX_ASSERT_PREEMPT_SAFE();
6933
6934 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6935 {
6936 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6937 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6938 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6939 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6940 }
6941
6942 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6943 VMMRZCallRing3Disable(pVCpu);
6944 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6945
6946 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6947 if (pVCpu->hm.s.Event.fPending)
6948 {
6949 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6950 Assert(!pVCpu->hm.s.Event.fPending);
6951 }
6952
6953 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6954 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6955
6956 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6957 and if we're injecting an event we should have a TRPM trap pending. */
6958 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6959#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6960 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6961#endif
6962
6963 /* Save guest state and restore host state bits. */
6964 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6965 AssertRCReturn(rc, rc);
6966 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6967 /* Thread-context hooks are unregistered at this point!!! */
6968
6969 /* Sync recompiler state. */
6970 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6971 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6972 | CPUM_CHANGED_LDTR
6973 | CPUM_CHANGED_GDTR
6974 | CPUM_CHANGED_IDTR
6975 | CPUM_CHANGED_TR
6976 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6977 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6978 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6979 {
6980 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6981 }
6982
6983 Assert(!pVCpu->hm.s.fClearTrapFlag);
6984
6985 /* Update the exit-to-ring 3 reason. */
6986 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6987
6988 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6989 if (rcExit != VINF_EM_RAW_INTERRUPT)
6990 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6991
6992 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6993
6994 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6995 VMMRZCallRing3RemoveNotification(pVCpu);
6996 VMMRZCallRing3Enable(pVCpu);
6997
6998 return rc;
6999}
7000
7001
7002/**
7003 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7004 * longjump to ring-3 and possibly get preempted.
7005 *
7006 * @returns VBox status code.
7007 * @param pVCpu The cross context virtual CPU structure.
7008 * @param enmOperation The operation causing the ring-3 longjump.
7009 * @param pvUser Opaque pointer to the guest-CPU context. The data
7010 * may be out-of-sync. Make sure to update the required
7011 * fields before using them.
7012 */
7013static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7014{
7015 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7016 {
7017 /*
7018 * !!! IMPORTANT !!!
7019 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7020 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7021 */
7022 VMMRZCallRing3RemoveNotification(pVCpu);
7023 VMMRZCallRing3Disable(pVCpu);
7024 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7025 RTThreadPreemptDisable(&PreemptState);
7026
7027 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7028 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7029
7030#if HC_ARCH_BITS == 64
7031 /* Restore host-state bits that VT-x only restores partially. */
7032 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7033 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7034 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7035 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7036#endif
7037
7038 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7039 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7040 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7041
7042 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7043 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7044 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7045 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7046 {
7047 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7048 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7049 }
7050
7051 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7052 VMMR0ThreadCtxHookDisable(pVCpu);
7053 HMR0LeaveCpu(pVCpu);
7054 RTThreadPreemptRestore(&PreemptState);
7055 return VINF_SUCCESS;
7056 }
7057
7058 Assert(pVCpu);
7059 Assert(pvUser);
7060 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7061 HMVMX_ASSERT_PREEMPT_SAFE();
7062
7063 VMMRZCallRing3Disable(pVCpu);
7064 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7065
7066 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7067
7068 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7069 AssertRCReturn(rc, rc);
7070
7071 VMMRZCallRing3Enable(pVCpu);
7072 return VINF_SUCCESS;
7073}
7074
7075
7076/**
7077 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7078 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7079 *
7080 * @param pVCpu The cross context virtual CPU structure.
7081 */
7082DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7083{
7084 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7085 {
7086 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7087 {
7088 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7089 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7090 AssertRC(rc);
7091 Log4Func(("Setup interrupt-window exiting\n"));
7092 }
7093 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7094}
7095
7096
7097/**
7098 * Clears the interrupt-window exiting control in the VMCS.
7099 *
7100 * @param pVCpu The cross context virtual CPU structure.
7101 */
7102DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7103{
7104 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7105 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7106 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7107 AssertRC(rc);
7108 Log4Func(("Cleared interrupt-window exiting\n"));
7109}
7110
7111
7112/**
7113 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7114 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7115 *
7116 * @param pVCpu The cross context virtual CPU structure.
7117 */
7118DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7119{
7120 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7121 {
7122 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7123 {
7124 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7125 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7126 AssertRC(rc);
7127 Log4Func(("Setup NMI-window exiting\n"));
7128 }
7129 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7130}
7131
7132
7133/**
7134 * Clears the NMI-window exiting control in the VMCS.
7135 *
7136 * @param pVCpu The cross context virtual CPU structure.
7137 */
7138DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7139{
7140 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7141 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7142 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7143 AssertRC(rc);
7144 Log4Func(("Cleared NMI-window exiting\n"));
7145}
7146
7147
7148/**
7149 * Evaluates the event to be delivered to the guest and sets it as the pending
7150 * event.
7151 *
7152 * @returns The VT-x guest-interruptibility state.
7153 * @param pVCpu The cross context virtual CPU structure.
7154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7155 * out-of-sync. Make sure to update the required fields
7156 * before using them.
7157 */
7158static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7159{
7160 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7161 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7162 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7163 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7164 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7165
7166 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7167 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7168 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7169 Assert(!TRPMHasTrap(pVCpu));
7170
7171 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7172 APICUpdatePendingInterrupts(pVCpu);
7173
7174 /*
7175 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7176 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7177 */
7178 /** @todo SMI. SMIs take priority over NMIs. */
7179 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7180 {
7181 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7182 if ( !pVCpu->hm.s.Event.fPending
7183 && !fBlockNmi
7184 && !fBlockSti
7185 && !fBlockMovSS)
7186 {
7187 Log4Func(("Pending NMI\n"));
7188 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7189 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7190
7191 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7192 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7193 }
7194 else
7195 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7196 }
7197 /*
7198 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7199 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7200 */
7201 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7202 && !pVCpu->hm.s.fSingleInstruction)
7203 {
7204 Assert(!DBGFIsStepping(pVCpu));
7205 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7206 AssertRCReturn(rc, 0);
7207 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7208 if ( !pVCpu->hm.s.Event.fPending
7209 && !fBlockInt
7210 && !fBlockSti
7211 && !fBlockMovSS)
7212 {
7213 uint8_t u8Interrupt;
7214 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7215 if (RT_SUCCESS(rc))
7216 {
7217 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7218 uint32_t u32IntInfo = u8Interrupt
7219 | VMX_EXIT_INTERRUPTION_INFO_VALID
7220 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7221
7222 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7223 }
7224 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7225 {
7226 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7227 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7228 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7229
7230 /*
7231 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7232 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7233 * need to re-set this force-flag here.
7234 */
7235 }
7236 else
7237 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7238 }
7239 else
7240 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7241 }
7242
7243 return fIntrState;
7244}
7245
7246
7247/**
7248 * Sets a pending-debug exception to be delivered to the guest if the guest is
7249 * single-stepping in the VMCS.
7250 *
7251 * @param pVCpu The cross context virtual CPU structure.
7252 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7253 * out-of-sync. Make sure to update the required fields
7254 * before using them.
7255 */
7256DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7257{
7258 RT_NOREF(pVCpu);
7259 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7260 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7261}
7262
7263
7264/**
7265 * Injects any pending events into the guest if the guest is in a state to
7266 * receive them.
7267 *
7268 * @returns Strict VBox status code (i.e. informational status codes too).
7269 * @param pVCpu The cross context virtual CPU structure.
7270 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7271 * out-of-sync. Make sure to update the required fields
7272 * before using them.
7273 * @param fIntrState The VT-x guest-interruptibility state.
7274 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7275 * return VINF_EM_DBG_STEPPED if the event was
7276 * dispatched directly.
7277 */
7278static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7279{
7280 HMVMX_ASSERT_PREEMPT_SAFE();
7281 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7282
7283 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7284 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7285
7286 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7287 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7288 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7289 Assert(!TRPMHasTrap(pVCpu));
7290
7291 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7292 if (pVCpu->hm.s.Event.fPending)
7293 {
7294 /*
7295 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7296 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7297 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7298 *
7299 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7300 */
7301 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7302#ifdef VBOX_STRICT
7303 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7304 {
7305 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7306 Assert(!fBlockInt);
7307 Assert(!fBlockSti);
7308 Assert(!fBlockMovSS);
7309 }
7310 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7311 {
7312 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7313 Assert(!fBlockSti);
7314 Assert(!fBlockMovSS);
7315 Assert(!fBlockNmi);
7316 }
7317#endif
7318 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7319 uIntType));
7320 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7321 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7322 &fIntrState);
7323 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7324
7325 /* Update the interruptibility-state as it could have been changed by
7326 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7327 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7328 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7329
7330 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7331 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7332 else
7333 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7334 }
7335
7336 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7337 if ( fBlockSti
7338 || fBlockMovSS)
7339 {
7340 if (!pVCpu->hm.s.fSingleInstruction)
7341 {
7342 /*
7343 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7344 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7345 * See Intel spec. 27.3.4 "Saving Non-Register State".
7346 */
7347 Assert(!DBGFIsStepping(pVCpu));
7348 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7349 AssertRCReturn(rc, rc);
7350 if (pMixedCtx->eflags.Bits.u1TF)
7351 {
7352 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7353 AssertRCReturn(rc2, rc2);
7354 }
7355 }
7356 else if (pMixedCtx->eflags.Bits.u1TF)
7357 {
7358 /*
7359 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7360 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7361 */
7362 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7363 fIntrState = 0;
7364 }
7365 }
7366
7367 /*
7368 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7369 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7370 */
7371 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7372 AssertRCReturn(rc3, rc3);
7373
7374 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7375 NOREF(fBlockMovSS); NOREF(fBlockSti);
7376 return rcStrict;
7377}
7378
7379
7380/**
7381 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7382 *
7383 * @param pVCpu The cross context virtual CPU structure.
7384 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7385 * out-of-sync. Make sure to update the required fields
7386 * before using them.
7387 */
7388DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7389{
7390 NOREF(pMixedCtx);
7391 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7392 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7393}
7394
7395
7396/**
7397 * Injects a double-fault (\#DF) exception into the VM.
7398 *
7399 * @returns Strict VBox status code (i.e. informational status codes too).
7400 * @param pVCpu The cross context virtual CPU structure.
7401 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7402 * out-of-sync. Make sure to update the required fields
7403 * before using them.
7404 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7405 * and should return VINF_EM_DBG_STEPPED if the event
7406 * is injected directly (register modified by us, not
7407 * by hardware on VM-entry).
7408 * @param pfIntrState Pointer to the current guest interruptibility-state.
7409 * This interruptibility-state will be updated if
7410 * necessary. This cannot not be NULL.
7411 */
7412DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7413{
7414 NOREF(pMixedCtx);
7415 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7416 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7417 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7418 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7419 pfIntrState);
7420}
7421
7422
7423/**
7424 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7425 *
7426 * @param pVCpu The cross context virtual CPU structure.
7427 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7428 * out-of-sync. Make sure to update the required fields
7429 * before using them.
7430 */
7431DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7432{
7433 NOREF(pMixedCtx);
7434 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7435 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7436 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7437}
7438
7439
7440/**
7441 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7442 *
7443 * @param pVCpu The cross context virtual CPU structure.
7444 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7445 * out-of-sync. Make sure to update the required fields
7446 * before using them.
7447 * @param cbInstr The value of RIP that is to be pushed on the guest
7448 * stack.
7449 */
7450DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7451{
7452 NOREF(pMixedCtx);
7453 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7454 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7455 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7456}
7457
7458
7459/**
7460 * Injects a general-protection (\#GP) fault into the VM.
7461 *
7462 * @returns Strict VBox status code (i.e. informational status codes too).
7463 * @param pVCpu The cross context virtual CPU structure.
7464 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7465 * out-of-sync. Make sure to update the required fields
7466 * before using them.
7467 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7468 * mode, i.e. in real-mode it's not valid).
7469 * @param u32ErrorCode The error code associated with the \#GP.
7470 * @param fStepping Whether we're running in
7471 * hmR0VmxRunGuestCodeStep() and should return
7472 * VINF_EM_DBG_STEPPED if the event is injected
7473 * directly (register modified by us, not by
7474 * hardware on VM-entry).
7475 * @param pfIntrState Pointer to the current guest interruptibility-state.
7476 * This interruptibility-state will be updated if
7477 * necessary. This cannot not be NULL.
7478 */
7479DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7480 bool fStepping, uint32_t *pfIntrState)
7481{
7482 NOREF(pMixedCtx);
7483 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7484 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485 if (fErrorCodeValid)
7486 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7487 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7488 pfIntrState);
7489}
7490
7491
7492#if 0 /* unused */
7493/**
7494 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7495 * VM.
7496 *
7497 * @param pVCpu The cross context virtual CPU structure.
7498 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7499 * out-of-sync. Make sure to update the required fields
7500 * before using them.
7501 * @param u32ErrorCode The error code associated with the \#GP.
7502 */
7503DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7504{
7505 NOREF(pMixedCtx);
7506 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7507 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7508 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7509 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7510}
7511#endif /* unused */
7512
7513
7514/**
7515 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7516 *
7517 * @param pVCpu The cross context virtual CPU structure.
7518 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7519 * out-of-sync. Make sure to update the required fields
7520 * before using them.
7521 * @param uVector The software interrupt vector number.
7522 * @param cbInstr The value of RIP that is to be pushed on the guest
7523 * stack.
7524 */
7525DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7526{
7527 NOREF(pMixedCtx);
7528 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7529 if ( uVector == X86_XCPT_BP
7530 || uVector == X86_XCPT_OF)
7531 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7532 else
7533 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7534 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7535}
7536
7537
7538/**
7539 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7540 * stack.
7541 *
7542 * @returns Strict VBox status code (i.e. informational status codes too).
7543 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7544 * @param pVM The cross context VM structure.
7545 * @param pMixedCtx Pointer to the guest-CPU context.
7546 * @param uValue The value to push to the guest stack.
7547 */
7548DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7549{
7550 /*
7551 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7552 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7553 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7554 */
7555 if (pMixedCtx->sp == 1)
7556 return VINF_EM_RESET;
7557 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7558 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7559 AssertRC(rc);
7560 return rc;
7561}
7562
7563
7564/**
7565 * Injects an event into the guest upon VM-entry by updating the relevant fields
7566 * in the VM-entry area in the VMCS.
7567 *
7568 * @returns Strict VBox status code (i.e. informational status codes too).
7569 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7570 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7571 *
7572 * @param pVCpu The cross context virtual CPU structure.
7573 * @param u64IntInfo The VM-entry interruption-information field.
7574 * @param cbInstr The VM-entry instruction length in bytes (for
7575 * software interrupts, exceptions and privileged
7576 * software exceptions).
7577 * @param u32ErrCode The VM-entry exception error code.
7578 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7579 * @param pfIntrState Pointer to the current guest interruptibility-state.
7580 * This interruptibility-state will be updated if
7581 * necessary. This cannot not be NULL.
7582 * @param fStepping Whether we're running in
7583 * hmR0VmxRunGuestCodeStep() and should return
7584 * VINF_EM_DBG_STEPPED if the event is injected
7585 * directly (register modified by us, not by
7586 * hardware on VM-entry).
7587 */
7588static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7589 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7590{
7591 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7592 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7593 Assert(pfIntrState);
7594
7595 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7596 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7597 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7598 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7599
7600#ifdef VBOX_STRICT
7601 /*
7602 * Validate the error-code-valid bit for hardware exceptions.
7603 * No error codes for exceptions in real-mode.
7604 *
7605 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7606 */
7607 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7608 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7609 {
7610 switch (uVector)
7611 {
7612 case X86_XCPT_PF:
7613 case X86_XCPT_DF:
7614 case X86_XCPT_TS:
7615 case X86_XCPT_NP:
7616 case X86_XCPT_SS:
7617 case X86_XCPT_GP:
7618 case X86_XCPT_AC:
7619 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7620 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7621 RT_FALL_THRU();
7622 default:
7623 break;
7624 }
7625 }
7626#endif
7627
7628 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7629 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7630 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7631
7632 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7633
7634 /*
7635 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7636 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7637 * interrupt handler in the (real-mode) guest.
7638 *
7639 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7640 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7641 */
7642 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7643 {
7644 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7645 {
7646 /*
7647 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7648 * set the deliver-error-code bit.
7649 *
7650 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7651 */
7652 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7653 }
7654 else
7655 {
7656 PVM pVM = pVCpu->CTX_SUFF(pVM);
7657 Assert(PDMVmmDevHeapIsEnabled(pVM));
7658 Assert(pVM->hm.s.vmx.pRealModeTSS);
7659
7660 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7661 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7662 | CPUMCTX_EXTRN_TABLE_MASK
7663 | CPUMCTX_EXTRN_RIP
7664 | CPUMCTX_EXTRN_RSP
7665 | CPUMCTX_EXTRN_RFLAGS);
7666 AssertRCReturn(rc2, rc2);
7667
7668 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7669 size_t const cbIdtEntry = sizeof(X86IDTR16);
7670 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7671 {
7672 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7673 if (uVector == X86_XCPT_DF)
7674 return VINF_EM_RESET;
7675
7676 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7677 if (uVector == X86_XCPT_GP)
7678 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7679
7680 /*
7681 * If we're injecting an event with no valid IDT entry, inject a #GP.
7682 * No error codes for exceptions in real-mode.
7683 *
7684 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7685 */
7686 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7687 pfIntrState);
7688 }
7689
7690 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7691 uint16_t uGuestIp = pMixedCtx->ip;
7692 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7693 {
7694 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7695 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7696 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7697 }
7698 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7699 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7700
7701 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7702 X86IDTR16 IdtEntry;
7703 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7704 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7705 AssertRCReturn(rc2, rc2);
7706
7707 /* Construct the stack frame for the interrupt/exception handler. */
7708 VBOXSTRICTRC rcStrict;
7709 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7710 if (rcStrict == VINF_SUCCESS)
7711 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7712 if (rcStrict == VINF_SUCCESS)
7713 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7714
7715 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7716 if (rcStrict == VINF_SUCCESS)
7717 {
7718 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7719 pMixedCtx->rip = IdtEntry.offSel;
7720 pMixedCtx->cs.Sel = IdtEntry.uSel;
7721 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7722 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7723 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7724 && uVector == X86_XCPT_PF)
7725 pMixedCtx->cr2 = GCPtrFaultAddress;
7726
7727 /* If any other guest-state bits are changed here, make sure to update
7728 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7729 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7730 | HM_CHANGED_GUEST_CR2
7731 | HM_CHANGED_GUEST_RIP
7732 | HM_CHANGED_GUEST_RFLAGS
7733 | HM_CHANGED_GUEST_RSP);
7734
7735 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7736 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7737 {
7738 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7739 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7740 Log4Func(("Clearing inhibition due to STI\n"));
7741 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7742 }
7743 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7744 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7745
7746 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7747 it, if we are returning to ring-3 before executing guest code. */
7748 pVCpu->hm.s.Event.fPending = false;
7749
7750 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7751 if (fStepping)
7752 rcStrict = VINF_EM_DBG_STEPPED;
7753 }
7754 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7755 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7756 return rcStrict;
7757 }
7758 }
7759
7760 /* Validate. */
7761 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7762 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7763 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7764
7765 /* Inject. */
7766 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7767 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7768 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7769 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7770 AssertRCReturn(rc, rc);
7771
7772 /* Update CR2. */
7773 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7774 && uVector == X86_XCPT_PF)
7775 pMixedCtx->cr2 = GCPtrFaultAddress;
7776
7777 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7778
7779 return VINF_SUCCESS;
7780}
7781
7782
7783/**
7784 * Clears the interrupt-window exiting control in the VMCS and if necessary
7785 * clears the current event in the VMCS as well.
7786 *
7787 * @returns VBox status code.
7788 * @param pVCpu The cross context virtual CPU structure.
7789 *
7790 * @remarks Use this function only to clear events that have not yet been
7791 * delivered to the guest but are injected in the VMCS!
7792 * @remarks No-long-jump zone!!!
7793 */
7794static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7795{
7796 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7797 {
7798 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7799 Log4Func(("Cleared interrupt widow\n"));
7800 }
7801
7802 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7803 {
7804 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7805 Log4Func(("Cleared interrupt widow\n"));
7806 }
7807}
7808
7809
7810/**
7811 * Enters the VT-x session.
7812 *
7813 * @returns VBox status code.
7814 * @param pVCpu The cross context virtual CPU structure.
7815 * @param pHostCpu Pointer to the global CPU info struct.
7816 */
7817VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7818{
7819 AssertPtr(pVCpu);
7820 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7821 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7822 RT_NOREF(pHostCpu);
7823
7824 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7825 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7826 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7827
7828#ifdef VBOX_STRICT
7829 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7830 RTCCUINTREG uHostCR4 = ASMGetCR4();
7831 if (!(uHostCR4 & X86_CR4_VMXE))
7832 {
7833 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7834 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7835 }
7836#endif
7837
7838 /*
7839 * Load the VCPU's VMCS as the current (and active) one.
7840 */
7841 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7842 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7843 if (RT_FAILURE(rc))
7844 return rc;
7845
7846 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7847 pVCpu->hm.s.fLeaveDone = false;
7848 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7849
7850 return VINF_SUCCESS;
7851}
7852
7853
7854/**
7855 * The thread-context callback (only on platforms which support it).
7856 *
7857 * @param enmEvent The thread-context event.
7858 * @param pVCpu The cross context virtual CPU structure.
7859 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7860 * @thread EMT(pVCpu)
7861 */
7862VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7863{
7864 NOREF(fGlobalInit);
7865
7866 switch (enmEvent)
7867 {
7868 case RTTHREADCTXEVENT_OUT:
7869 {
7870 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7871 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7872 VMCPU_ASSERT_EMT(pVCpu);
7873
7874 /* No longjmps (logger flushes, locks) in this fragile context. */
7875 VMMRZCallRing3Disable(pVCpu);
7876 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7877
7878 /*
7879 * Restore host-state (FPU, debug etc.)
7880 */
7881 if (!pVCpu->hm.s.fLeaveDone)
7882 {
7883 /*
7884 * Do -not- import the guest-state here as we might already be in the middle of importing
7885 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7886 */
7887 hmR0VmxLeave(pVCpu, false /* fImportState */);
7888 pVCpu->hm.s.fLeaveDone = true;
7889 }
7890
7891 /* Leave HM context, takes care of local init (term). */
7892 int rc = HMR0LeaveCpu(pVCpu);
7893 AssertRC(rc); NOREF(rc);
7894
7895 /* Restore longjmp state. */
7896 VMMRZCallRing3Enable(pVCpu);
7897 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7898 break;
7899 }
7900
7901 case RTTHREADCTXEVENT_IN:
7902 {
7903 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7904 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7905 VMCPU_ASSERT_EMT(pVCpu);
7906
7907 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7908 VMMRZCallRing3Disable(pVCpu);
7909 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7910
7911 /* Initialize the bare minimum state required for HM. This takes care of
7912 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7913 int rc = hmR0EnterCpu(pVCpu);
7914 AssertRC(rc);
7915 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7916 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7917
7918 /* Load the active VMCS as the current one. */
7919 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7920 {
7921 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7922 AssertRC(rc); NOREF(rc);
7923 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7924 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7925 }
7926 pVCpu->hm.s.fLeaveDone = false;
7927
7928 /* Restore longjmp state. */
7929 VMMRZCallRing3Enable(pVCpu);
7930 break;
7931 }
7932
7933 default:
7934 break;
7935 }
7936}
7937
7938
7939/**
7940 * Exports the host state into the VMCS host-state area.
7941 * Sets up the VM-exit MSR-load area.
7942 *
7943 * The CPU state will be loaded from these fields on every successful VM-exit.
7944 *
7945 * @returns VBox status code.
7946 * @param pVCpu The cross context virtual CPU structure.
7947 *
7948 * @remarks No-long-jump zone!!!
7949 */
7950static int hmR0VmxExportHostState(PVMCPU pVCpu)
7951{
7952 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7953
7954 int rc = VINF_SUCCESS;
7955 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7956 {
7957 rc = hmR0VmxExportHostControlRegs();
7958 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7959
7960 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7961 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7962
7963 rc = hmR0VmxExportHostMsrs(pVCpu);
7964 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7965
7966 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7967 }
7968 return rc;
7969}
7970
7971
7972/**
7973 * Saves the host state in the VMCS host-state.
7974 *
7975 * @returns VBox status code.
7976 * @param pVCpu The cross context virtual CPU structure.
7977 *
7978 * @remarks No-long-jump zone!!!
7979 */
7980VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7981{
7982 AssertPtr(pVCpu);
7983 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7984
7985 /*
7986 * Export the host state here while entering HM context.
7987 * When thread-context hooks are used, we might get preempted and have to re-save the host
7988 * state but most of the time we won't be, so do it here before we disable interrupts.
7989 */
7990 return hmR0VmxExportHostState(pVCpu);
7991}
7992
7993
7994/**
7995 * Exports the guest state into the VMCS guest-state area.
7996 *
7997 * The will typically be done before VM-entry when the guest-CPU state and the
7998 * VMCS state may potentially be out of sync.
7999 *
8000 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8001 * VM-entry controls.
8002 * Sets up the appropriate VMX non-root function to execute guest code based on
8003 * the guest CPU mode.
8004 *
8005 * @returns VBox strict status code.
8006 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8007 * without unrestricted guest access and the VMMDev is not presently
8008 * mapped (e.g. EFI32).
8009 *
8010 * @param pVCpu The cross context virtual CPU structure.
8011 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8012 * out-of-sync. Make sure to update the required fields
8013 * before using them.
8014 *
8015 * @remarks No-long-jump zone!!!
8016 */
8017static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8018{
8019 AssertPtr(pVCpu);
8020 AssertPtr(pMixedCtx);
8021 HMVMX_ASSERT_PREEMPT_SAFE();
8022
8023 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8024
8025 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8026
8027 /* Determine real-on-v86 mode. */
8028 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8029 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8030 && CPUMIsGuestInRealModeEx(pMixedCtx))
8031 {
8032 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8033 }
8034
8035 /*
8036 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8037 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8038 */
8039 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8040 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8041
8042 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8043 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8044 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8045
8046 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8047 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8048 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8049
8050 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8051 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8052
8053 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8054 if (rcStrict == VINF_SUCCESS)
8055 { /* likely */ }
8056 else
8057 {
8058 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8059 return rcStrict;
8060 }
8061
8062 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8063 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8064
8065 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8066 may alter controls if we determine we don't have to swap EFER after all. */
8067 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8068 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8069
8070 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8071 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8072
8073 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8074 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8075 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8076
8077 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8078 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8079 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8080 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8081 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8082 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8083
8084 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8085 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8086 | HM_CHANGED_GUEST_CR2
8087 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8088 | HM_CHANGED_GUEST_X87
8089 | HM_CHANGED_GUEST_SSE_AVX
8090 | HM_CHANGED_GUEST_OTHER_XSAVE
8091 | HM_CHANGED_GUEST_XCRx
8092 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8093 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8094 | HM_CHANGED_GUEST_TSC_AUX
8095 | HM_CHANGED_GUEST_OTHER_MSRS
8096 | HM_CHANGED_GUEST_HWVIRT
8097 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8098
8099 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8100 return rc;
8101}
8102
8103
8104/**
8105 * Exports the state shared between the host and guest into the VMCS.
8106 *
8107 * @param pVCpu The cross context virtual CPU structure.
8108 * @param pCtx Pointer to the guest-CPU context.
8109 *
8110 * @remarks No-long-jump zone!!!
8111 */
8112static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8113{
8114 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8115 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8116
8117 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8118 {
8119 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8120 AssertRC(rc);
8121 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8122
8123 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8124 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8125 {
8126 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8127 AssertRC(rc);
8128 }
8129 }
8130
8131 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8132 {
8133 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8134 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8135 }
8136
8137 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8138 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8139}
8140
8141
8142/**
8143 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8144 *
8145 * @returns Strict VBox status code (i.e. informational status codes too).
8146 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8147 * without unrestricted guest access and the VMMDev is not presently
8148 * mapped (e.g. EFI32).
8149 *
8150 * @param pVCpu The cross context virtual CPU structure.
8151 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8152 * out-of-sync. Make sure to update the required fields
8153 * before using them.
8154 *
8155 * @remarks No-long-jump zone!!!
8156 */
8157static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8158{
8159 HMVMX_ASSERT_PREEMPT_SAFE();
8160 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8161 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8162
8163#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8164 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8165#endif
8166
8167 /*
8168 * For many exits it's only RIP that changes and hence try to export it first
8169 * without going through a lot of change flag checks.
8170 */
8171 VBOXSTRICTRC rcStrict;
8172 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8173 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8174 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8175 {
8176 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8177 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8178 { /* likely */}
8179 else
8180 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8181 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8182 }
8183 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8184 {
8185 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8186 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8187 { /* likely */}
8188 else
8189 {
8190 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8191 VBOXSTRICTRC_VAL(rcStrict)));
8192 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8193 return rcStrict;
8194 }
8195 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8196 }
8197 else
8198 rcStrict = VINF_SUCCESS;
8199
8200#ifdef VBOX_STRICT
8201 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8202 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8203 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8204 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8205 ("fCtxChanged=%#RX64\n", fCtxChanged));
8206#endif
8207 return rcStrict;
8208}
8209
8210
8211/**
8212 * Does the preparations before executing guest code in VT-x.
8213 *
8214 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8215 * recompiler/IEM. We must be cautious what we do here regarding committing
8216 * guest-state information into the VMCS assuming we assuredly execute the
8217 * guest in VT-x mode.
8218 *
8219 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8220 * the common-state (TRPM/forceflags), we must undo those changes so that the
8221 * recompiler/IEM can (and should) use them when it resumes guest execution.
8222 * Otherwise such operations must be done when we can no longer exit to ring-3.
8223 *
8224 * @returns Strict VBox status code (i.e. informational status codes too).
8225 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8226 * have been disabled.
8227 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8228 * double-fault into the guest.
8229 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8230 * dispatched directly.
8231 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8232 *
8233 * @param pVCpu The cross context virtual CPU structure.
8234 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8235 * out-of-sync. Make sure to update the required fields
8236 * before using them.
8237 * @param pVmxTransient Pointer to the VMX transient structure.
8238 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8239 * us ignore some of the reasons for returning to
8240 * ring-3, and return VINF_EM_DBG_STEPPED if event
8241 * dispatching took place.
8242 */
8243static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8244{
8245 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8246
8247#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8248 PGMRZDynMapFlushAutoSet(pVCpu);
8249#endif
8250
8251 /* Check force flag actions that might require us to go back to ring-3. */
8252 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8253 if (rcStrict == VINF_SUCCESS)
8254 { /* FFs doesn't get set all the time. */ }
8255 else
8256 return rcStrict;
8257
8258 /*
8259 * Setup the virtualized-APIC accesses.
8260 *
8261 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8262 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8263 *
8264 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8265 */
8266 PVM pVM = pVCpu->CTX_SUFF(pVM);
8267 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8268 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8269 && PDMHasApic(pVM))
8270 {
8271 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8272 Assert(u64MsrApicBase);
8273 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8274
8275 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8276
8277 /* Unalias any existing mapping. */
8278 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8279 AssertRCReturn(rc, rc);
8280
8281 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8282 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8283 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8284 AssertRCReturn(rc, rc);
8285
8286 /* Update the per-VCPU cache of the APIC base MSR. */
8287 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8288 }
8289
8290 if (TRPMHasTrap(pVCpu))
8291 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8292 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8293
8294 /*
8295 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8296 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8297 * also result in triple-faulting the VM.
8298 */
8299 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8300 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8301 { /* likely */ }
8302 else
8303 {
8304 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8305 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8306 return rcStrict;
8307 }
8308
8309 /*
8310 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8311 * import CR3 themselves. We will need to update them here as even as late as the above
8312 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8313 * the below force flags to be set.
8314 */
8315 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8316 {
8317 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8318 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8319 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8320 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8321 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8322 }
8323 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8324 {
8325 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8326 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8327 }
8328
8329 /*
8330 * No longjmps to ring-3 from this point on!!!
8331 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8332 * This also disables flushing of the R0-logger instance (if any).
8333 */
8334 VMMRZCallRing3Disable(pVCpu);
8335
8336 /*
8337 * Export the guest state bits.
8338 *
8339 * We cannot perform longjmps while loading the guest state because we do not preserve the
8340 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8341 * CPU migration.
8342 *
8343 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8344 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8345 * Hence, loading of the guest state needs to be done -after- injection of events.
8346 */
8347 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8348 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8349 { /* likely */ }
8350 else
8351 {
8352 VMMRZCallRing3Enable(pVCpu);
8353 return rcStrict;
8354 }
8355
8356 /*
8357 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8358 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8359 * preemption disabled for a while. Since this is purly to aid the
8360 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8361 * disable interrupt on NT.
8362 *
8363 * We need to check for force-flags that could've possible been altered since we last
8364 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8365 * see @bugref{6398}).
8366 *
8367 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8368 * to ring-3 before executing guest code.
8369 */
8370 pVmxTransient->fEFlags = ASMIntDisableFlags();
8371
8372 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8373 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8374 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8375 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8376 {
8377 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8378 {
8379 pVCpu->hm.s.Event.fPending = false;
8380
8381 /*
8382 * We've injected any pending events. This is really the point of no return (to ring-3).
8383 *
8384 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8385 * returns from this function, so don't enable them here.
8386 */
8387 return VINF_SUCCESS;
8388 }
8389
8390 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8391 rcStrict = VINF_EM_RAW_INTERRUPT;
8392 }
8393 else
8394 {
8395 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8396 rcStrict = VINF_EM_RAW_TO_R3;
8397 }
8398
8399 ASMSetFlags(pVmxTransient->fEFlags);
8400 VMMRZCallRing3Enable(pVCpu);
8401
8402 return rcStrict;
8403}
8404
8405
8406/**
8407 * Prepares to run guest code in VT-x and we've committed to doing so. This
8408 * means there is no backing out to ring-3 or anywhere else at this
8409 * point.
8410 *
8411 * @param pVCpu The cross context virtual CPU structure.
8412 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8413 * out-of-sync. Make sure to update the required fields
8414 * before using them.
8415 * @param pVmxTransient Pointer to the VMX transient structure.
8416 *
8417 * @remarks Called with preemption disabled.
8418 * @remarks No-long-jump zone!!!
8419 */
8420static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8421{
8422 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8423 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8424 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8425
8426 /*
8427 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8428 */
8429 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8430 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8431
8432 PVM pVM = pVCpu->CTX_SUFF(pVM);
8433 if (!CPUMIsGuestFPUStateActive(pVCpu))
8434 {
8435 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8436 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8437 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8438 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8439 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8440 }
8441
8442 /*
8443 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8444 */
8445 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8446 && pVCpu->hm.s.vmx.cMsrs > 0)
8447 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8448
8449 /*
8450 * Re-save the host state bits as we may've been preempted (only happens when
8451 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8452 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8453 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8454 * See @bugref{8432}.
8455 */
8456 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8457 {
8458 int rc = hmR0VmxExportHostState(pVCpu);
8459 AssertRC(rc);
8460 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8461 }
8462 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8463
8464 /*
8465 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8466 */
8467 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8468 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8469 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8470
8471 /* Store status of the shared guest-host state at the time of VM-entry. */
8472#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8473 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8474 {
8475 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8476 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8477 }
8478 else
8479#endif
8480 {
8481 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8482 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8483 }
8484
8485 /*
8486 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8487 */
8488 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8489 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8490
8491 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8492 RTCPUID idCurrentCpu = pCpu->idCpu;
8493 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8494 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8495 {
8496 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8497 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8498 }
8499
8500 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8501 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8502 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8503 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8504
8505 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8506
8507 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8508 to start executing. */
8509
8510 /*
8511 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8512 */
8513 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8514 {
8515 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8516 {
8517 bool fMsrUpdated;
8518 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8519 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8520 &fMsrUpdated);
8521 AssertRC(rc2);
8522 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8523 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8524 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8525 }
8526 else
8527 {
8528 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8529 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8530 }
8531 }
8532
8533 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8534 {
8535 bool fMsrUpdated;
8536 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8537 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8538 &fMsrUpdated);
8539 AssertRC(rc2);
8540 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8541 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8542 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8543 }
8544
8545#ifdef VBOX_STRICT
8546 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8547 hmR0VmxCheckHostEferMsr(pVCpu);
8548 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8549#endif
8550#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8551 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8552 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8553 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8554#endif
8555}
8556
8557
8558/**
8559 * Performs some essential restoration of state after running guest code in
8560 * VT-x.
8561 *
8562 * @param pVCpu The cross context virtual CPU structure.
8563 * @param pVmxTransient Pointer to the VMX transient structure.
8564 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8565 *
8566 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8567 *
8568 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8569 * unconditionally when it is safe to do so.
8570 */
8571static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8572{
8573 uint64_t const uHostTsc = ASMReadTSC();
8574 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8575
8576 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8577 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8578 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8579 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8580 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8581 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8582
8583 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8584 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8585
8586 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8587 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8588 Assert(!ASMIntAreEnabled());
8589 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8590
8591#if HC_ARCH_BITS == 64
8592 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8593#endif
8594#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8595 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8596 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8597 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8598#else
8599 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8600#endif
8601#ifdef VBOX_STRICT
8602 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8603#endif
8604 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8605
8606 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8607 uint32_t uExitReason;
8608 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8609 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8610 AssertRC(rc);
8611 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8612 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8613
8614 if (rcVMRun == VINF_SUCCESS)
8615 {
8616 /*
8617 * Update the VM-exit history array here even if the VM-entry failed due to:
8618 * - Invalid guest state.
8619 * - MSR loading.
8620 * - Machine-check event.
8621 *
8622 * In any of the above cases we will still have a "valid" VM-exit reason
8623 * despite @a fVMEntryFailed being false.
8624 *
8625 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8626 *
8627 * Note! We don't have CS or RIP at this point. Will probably address that later
8628 * by amending the history entry added here.
8629 */
8630 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8631 UINT64_MAX, uHostTsc);
8632
8633 if (!pVmxTransient->fVMEntryFailed)
8634 {
8635 VMMRZCallRing3Enable(pVCpu);
8636
8637 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8638 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8639
8640#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8641 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8642 AssertRC(rc);
8643#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8644 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8645 AssertRC(rc);
8646#else
8647 /*
8648 * Import the guest-interruptibility state always as we need it while evaluating
8649 * injecting events on re-entry.
8650 *
8651 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8652 * checking for real-mode while exporting the state because all bits that cause
8653 * mode changes wrt CR0 are intercepted.
8654 */
8655 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8656 AssertRC(rc);
8657#endif
8658
8659 /*
8660 * Sync the TPR shadow with our APIC state.
8661 */
8662 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8663 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8664 {
8665 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8666 AssertRC(rc);
8667 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8668 }
8669
8670 return;
8671 }
8672 }
8673 else
8674 {
8675 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8676 }
8677
8678 VMMRZCallRing3Enable(pVCpu);
8679}
8680
8681
8682/**
8683 * Runs the guest code using VT-x the normal way.
8684 *
8685 * @returns VBox status code.
8686 * @param pVCpu The cross context virtual CPU structure.
8687 * @param pCtx Pointer to the guest-CPU context.
8688 *
8689 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8690 */
8691static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8692{
8693 VMXTRANSIENT VmxTransient;
8694 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8695 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8696 uint32_t cLoops = 0;
8697
8698 for (;; cLoops++)
8699 {
8700 Assert(!HMR0SuspendPending());
8701 HMVMX_ASSERT_CPU_SAFE();
8702
8703 /* Preparatory work for running guest code, this may force us to return
8704 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8705 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8706 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8707 if (rcStrict != VINF_SUCCESS)
8708 break;
8709
8710 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8711 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8712 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8713
8714 /* Restore any residual host-state and save any bits shared between host
8715 and guest into the guest-CPU state. Re-enables interrupts! */
8716 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8717
8718 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8719 if (RT_SUCCESS(rcRun))
8720 { /* very likely */ }
8721 else
8722 {
8723 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8724 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8725 return rcRun;
8726 }
8727
8728 /* Profile the VM-exit. */
8729 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8730 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8731 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8732 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8733 HMVMX_START_EXIT_DISPATCH_PROF();
8734
8735 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8736
8737 /* Handle the VM-exit. */
8738#ifdef HMVMX_USE_FUNCTION_TABLE
8739 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8740#else
8741 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8742#endif
8743 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8744 if (rcStrict == VINF_SUCCESS)
8745 {
8746 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8747 continue; /* likely */
8748 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8749 rcStrict = VINF_EM_RAW_INTERRUPT;
8750 }
8751 break;
8752 }
8753
8754 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8755 return rcStrict;
8756}
8757
8758
8759
8760/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8761 * probes.
8762 *
8763 * The following few functions and associated structure contains the bloat
8764 * necessary for providing detailed debug events and dtrace probes as well as
8765 * reliable host side single stepping. This works on the principle of
8766 * "subclassing" the normal execution loop and workers. We replace the loop
8767 * method completely and override selected helpers to add necessary adjustments
8768 * to their core operation.
8769 *
8770 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8771 * any performance for debug and analysis features.
8772 *
8773 * @{
8774 */
8775
8776/**
8777 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8778 * the debug run loop.
8779 */
8780typedef struct VMXRUNDBGSTATE
8781{
8782 /** The RIP we started executing at. This is for detecting that we stepped. */
8783 uint64_t uRipStart;
8784 /** The CS we started executing with. */
8785 uint16_t uCsStart;
8786
8787 /** Whether we've actually modified the 1st execution control field. */
8788 bool fModifiedProcCtls : 1;
8789 /** Whether we've actually modified the 2nd execution control field. */
8790 bool fModifiedProcCtls2 : 1;
8791 /** Whether we've actually modified the exception bitmap. */
8792 bool fModifiedXcptBitmap : 1;
8793
8794 /** We desire the modified the CR0 mask to be cleared. */
8795 bool fClearCr0Mask : 1;
8796 /** We desire the modified the CR4 mask to be cleared. */
8797 bool fClearCr4Mask : 1;
8798 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8799 uint32_t fCpe1Extra;
8800 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8801 uint32_t fCpe1Unwanted;
8802 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8803 uint32_t fCpe2Extra;
8804 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8805 uint32_t bmXcptExtra;
8806 /** The sequence number of the Dtrace provider settings the state was
8807 * configured against. */
8808 uint32_t uDtraceSettingsSeqNo;
8809 /** VM-exits to check (one bit per VM-exit). */
8810 uint32_t bmExitsToCheck[3];
8811
8812 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8813 uint32_t fProcCtlsInitial;
8814 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8815 uint32_t fProcCtls2Initial;
8816 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8817 uint32_t bmXcptInitial;
8818} VMXRUNDBGSTATE;
8819AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8820typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8821
8822
8823/**
8824 * Initializes the VMXRUNDBGSTATE structure.
8825 *
8826 * @param pVCpu The cross context virtual CPU structure of the
8827 * calling EMT.
8828 * @param pCtx The CPU register context to go with @a pVCpu.
8829 * @param pDbgState The structure to initialize.
8830 */
8831static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8832{
8833 pDbgState->uRipStart = pCtx->rip;
8834 pDbgState->uCsStart = pCtx->cs.Sel;
8835
8836 pDbgState->fModifiedProcCtls = false;
8837 pDbgState->fModifiedProcCtls2 = false;
8838 pDbgState->fModifiedXcptBitmap = false;
8839 pDbgState->fClearCr0Mask = false;
8840 pDbgState->fClearCr4Mask = false;
8841 pDbgState->fCpe1Extra = 0;
8842 pDbgState->fCpe1Unwanted = 0;
8843 pDbgState->fCpe2Extra = 0;
8844 pDbgState->bmXcptExtra = 0;
8845 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8846 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8847 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8848}
8849
8850
8851/**
8852 * Updates the VMSC fields with changes requested by @a pDbgState.
8853 *
8854 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8855 * immediately before executing guest code, i.e. when interrupts are disabled.
8856 * We don't check status codes here as we cannot easily assert or return in the
8857 * latter case.
8858 *
8859 * @param pVCpu The cross context virtual CPU structure.
8860 * @param pDbgState The debug state.
8861 */
8862static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8863{
8864 /*
8865 * Ensure desired flags in VMCS control fields are set.
8866 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8867 *
8868 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8869 * there should be no stale data in pCtx at this point.
8870 */
8871 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8872 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8873 {
8874 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8875 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8876 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8877 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8878 pDbgState->fModifiedProcCtls = true;
8879 }
8880
8881 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8882 {
8883 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8884 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8885 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8886 pDbgState->fModifiedProcCtls2 = true;
8887 }
8888
8889 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8890 {
8891 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8892 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8893 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8894 pDbgState->fModifiedXcptBitmap = true;
8895 }
8896
8897 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8898 {
8899 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8900 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8901 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8902 }
8903
8904 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8905 {
8906 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8907 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8908 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8909 }
8910}
8911
8912
8913static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8914{
8915 /*
8916 * Restore VM-exit control settings as we may not reenter this function the
8917 * next time around.
8918 */
8919 /* We reload the initial value, trigger what we can of recalculations the
8920 next time around. From the looks of things, that's all that's required atm. */
8921 if (pDbgState->fModifiedProcCtls)
8922 {
8923 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8924 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8925 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8926 AssertRCReturn(rc2, rc2);
8927 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8928 }
8929
8930 /* We're currently the only ones messing with this one, so just restore the
8931 cached value and reload the field. */
8932 if ( pDbgState->fModifiedProcCtls2
8933 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8934 {
8935 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8936 AssertRCReturn(rc2, rc2);
8937 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8938 }
8939
8940 /* If we've modified the exception bitmap, we restore it and trigger
8941 reloading and partial recalculation the next time around. */
8942 if (pDbgState->fModifiedXcptBitmap)
8943 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8944
8945 return rcStrict;
8946}
8947
8948
8949/**
8950 * Configures VM-exit controls for current DBGF and DTrace settings.
8951 *
8952 * This updates @a pDbgState and the VMCS execution control fields to reflect
8953 * the necessary VM-exits demanded by DBGF and DTrace.
8954 *
8955 * @param pVCpu The cross context virtual CPU structure.
8956 * @param pDbgState The debug state.
8957 * @param pVmxTransient Pointer to the VMX transient structure. May update
8958 * fUpdateTscOffsettingAndPreemptTimer.
8959 */
8960static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8961{
8962 /*
8963 * Take down the dtrace serial number so we can spot changes.
8964 */
8965 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8966 ASMCompilerBarrier();
8967
8968 /*
8969 * We'll rebuild most of the middle block of data members (holding the
8970 * current settings) as we go along here, so start by clearing it all.
8971 */
8972 pDbgState->bmXcptExtra = 0;
8973 pDbgState->fCpe1Extra = 0;
8974 pDbgState->fCpe1Unwanted = 0;
8975 pDbgState->fCpe2Extra = 0;
8976 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8977 pDbgState->bmExitsToCheck[i] = 0;
8978
8979 /*
8980 * Software interrupts (INT XXh) - no idea how to trigger these...
8981 */
8982 PVM pVM = pVCpu->CTX_SUFF(pVM);
8983 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8984 || VBOXVMM_INT_SOFTWARE_ENABLED())
8985 {
8986 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8987 }
8988
8989 /*
8990 * INT3 breakpoints - triggered by #BP exceptions.
8991 */
8992 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8993 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8994
8995 /*
8996 * Exception bitmap and XCPT events+probes.
8997 */
8998 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8999 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9000 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9001
9002 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9003 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9004 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9005 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9006 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9007 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9008 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9009 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9010 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9011 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9012 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9013 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9014 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9015 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9016 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9017 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9018 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9019 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9020
9021 if (pDbgState->bmXcptExtra)
9022 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9023
9024 /*
9025 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9026 *
9027 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9028 * So, when adding/changing/removing please don't forget to update it.
9029 *
9030 * Some of the macros are picking up local variables to save horizontal space,
9031 * (being able to see it in a table is the lesser evil here).
9032 */
9033#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9034 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9035 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9036#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9037 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9038 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9039 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9040 } else do { } while (0)
9041#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9042 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9043 { \
9044 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9045 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9046 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9047 } else do { } while (0)
9048#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9049 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9050 { \
9051 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9052 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9053 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9054 } else do { } while (0)
9055#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9056 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9057 { \
9058 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9059 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9060 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9061 } else do { } while (0)
9062
9063 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9064 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9065 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9066 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9067 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9068
9069 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9070 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9071 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9072 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9073 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9074 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9075 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9076 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9077 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9078 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9079 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9080 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9081 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9082 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9083 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9084 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9085 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9086 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9087 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9088 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9089 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9090 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9091 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9092 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9093 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9094 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9095 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9096 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9097 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9098 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9099 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9101 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9102 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9103 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9104 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9105
9106 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9107 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9108 {
9109 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9110 | CPUMCTX_EXTRN_CR4
9111 | CPUMCTX_EXTRN_APIC_TPR);
9112 AssertRC(rc);
9113
9114#if 0 /** @todo fix me */
9115 pDbgState->fClearCr0Mask = true;
9116 pDbgState->fClearCr4Mask = true;
9117#endif
9118 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9119 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9120 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9121 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9122 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9123 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9124 require clearing here and in the loop if we start using it. */
9125 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9126 }
9127 else
9128 {
9129 if (pDbgState->fClearCr0Mask)
9130 {
9131 pDbgState->fClearCr0Mask = false;
9132 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9133 }
9134 if (pDbgState->fClearCr4Mask)
9135 {
9136 pDbgState->fClearCr4Mask = false;
9137 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9138 }
9139 }
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9141 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9142
9143 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9144 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9145 {
9146 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9147 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9148 }
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9151
9152 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9154 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9156 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9158 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9160#if 0 /** @todo too slow, fix handler. */
9161 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9162#endif
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9164
9165 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9166 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9167 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9168 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9169 {
9170 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9171 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9172 }
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9174 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9176 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9177
9178 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9179 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9180 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9181 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9182 {
9183 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9184 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9185 }
9186 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9187 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9188 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9189 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9190
9191 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9192 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9193 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9194 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9195 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9196 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9197 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9198 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9199 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9200 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9201 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9202 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9203 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9204 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9205 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9206 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9207 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9208 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9209 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9210 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9211 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9212 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9213
9214#undef IS_EITHER_ENABLED
9215#undef SET_ONLY_XBM_IF_EITHER_EN
9216#undef SET_CPE1_XBM_IF_EITHER_EN
9217#undef SET_CPEU_XBM_IF_EITHER_EN
9218#undef SET_CPE2_XBM_IF_EITHER_EN
9219
9220 /*
9221 * Sanitize the control stuff.
9222 */
9223 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9224 if (pDbgState->fCpe2Extra)
9225 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9226 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9227 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9228 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9229 {
9230 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9231 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9232 }
9233
9234 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9235 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9236 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9237 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9238}
9239
9240
9241/**
9242 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9243 * appropriate.
9244 *
9245 * The caller has checked the VM-exit against the
9246 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9247 * already, so we don't have to do that either.
9248 *
9249 * @returns Strict VBox status code (i.e. informational status codes too).
9250 * @param pVCpu The cross context virtual CPU structure.
9251 * @param pMixedCtx Pointer to the guest-CPU context.
9252 * @param pVmxTransient Pointer to the VMX-transient structure.
9253 * @param uExitReason The VM-exit reason.
9254 *
9255 * @remarks The name of this function is displayed by dtrace, so keep it short
9256 * and to the point. No longer than 33 chars long, please.
9257 */
9258static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9259 uint32_t uExitReason)
9260{
9261 /*
9262 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9263 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9264 *
9265 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9266 * does. Must add/change/remove both places. Same ordering, please.
9267 *
9268 * Added/removed events must also be reflected in the next section
9269 * where we dispatch dtrace events.
9270 */
9271 bool fDtrace1 = false;
9272 bool fDtrace2 = false;
9273 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9274 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9275 uint32_t uEventArg = 0;
9276#define SET_EXIT(a_EventSubName) \
9277 do { \
9278 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9279 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9280 } while (0)
9281#define SET_BOTH(a_EventSubName) \
9282 do { \
9283 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9284 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9285 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9286 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9287 } while (0)
9288 switch (uExitReason)
9289 {
9290 case VMX_EXIT_MTF:
9291 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9292
9293 case VMX_EXIT_XCPT_OR_NMI:
9294 {
9295 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9296 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9297 {
9298 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9299 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9300 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9301 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9302 {
9303 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9304 {
9305 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9306 uEventArg = pVmxTransient->uExitIntErrorCode;
9307 }
9308 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9309 switch (enmEvent1)
9310 {
9311 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9312 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9313 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9314 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9315 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9316 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9317 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9318 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9319 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9320 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9321 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9322 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9323 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9324 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9325 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9326 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9327 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9328 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9329 default: break;
9330 }
9331 }
9332 else
9333 AssertFailed();
9334 break;
9335
9336 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9337 uEventArg = idxVector;
9338 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9339 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9340 break;
9341 }
9342 break;
9343 }
9344
9345 case VMX_EXIT_TRIPLE_FAULT:
9346 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9347 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9348 break;
9349 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9350 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9351 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9352 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9353 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9354
9355 /* Instruction specific VM-exits: */
9356 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9357 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9358 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9359 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9360 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9361 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9362 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9363 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9364 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9365 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9366 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9367 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9368 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9369 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9370 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9371 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9372 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9373 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9374 case VMX_EXIT_MOV_CRX:
9375 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9376 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9377 SET_BOTH(CRX_READ);
9378 else
9379 SET_BOTH(CRX_WRITE);
9380 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9381 break;
9382 case VMX_EXIT_MOV_DRX:
9383 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9384 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9385 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9386 SET_BOTH(DRX_READ);
9387 else
9388 SET_BOTH(DRX_WRITE);
9389 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9390 break;
9391 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9392 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9393 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9394 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9395 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9396 case VMX_EXIT_XDTR_ACCESS:
9397 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9398 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9399 {
9400 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9401 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9402 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9403 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9404 }
9405 break;
9406
9407 case VMX_EXIT_TR_ACCESS:
9408 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9409 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9410 {
9411 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9412 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9413 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9414 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9415 }
9416 break;
9417
9418 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9419 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9420 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9421 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9422 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9423 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9424 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9425 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9426 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9427 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9428 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9429
9430 /* Events that aren't relevant at this point. */
9431 case VMX_EXIT_EXT_INT:
9432 case VMX_EXIT_INT_WINDOW:
9433 case VMX_EXIT_NMI_WINDOW:
9434 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9435 case VMX_EXIT_PREEMPT_TIMER:
9436 case VMX_EXIT_IO_INSTR:
9437 break;
9438
9439 /* Errors and unexpected events. */
9440 case VMX_EXIT_INIT_SIGNAL:
9441 case VMX_EXIT_SIPI:
9442 case VMX_EXIT_IO_SMI:
9443 case VMX_EXIT_SMI:
9444 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9445 case VMX_EXIT_ERR_MSR_LOAD:
9446 case VMX_EXIT_ERR_MACHINE_CHECK:
9447 break;
9448
9449 default:
9450 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9451 break;
9452 }
9453#undef SET_BOTH
9454#undef SET_EXIT
9455
9456 /*
9457 * Dtrace tracepoints go first. We do them here at once so we don't
9458 * have to copy the guest state saving and stuff a few dozen times.
9459 * Down side is that we've got to repeat the switch, though this time
9460 * we use enmEvent since the probes are a subset of what DBGF does.
9461 */
9462 if (fDtrace1 || fDtrace2)
9463 {
9464 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9465 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9466 switch (enmEvent1)
9467 {
9468 /** @todo consider which extra parameters would be helpful for each probe. */
9469 case DBGFEVENT_END: break;
9470 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9471 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9472 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9473 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9475 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9476 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9477 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9478 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9479 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9480 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9481 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9482 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9483 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9484 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9485 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9486 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9488 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9489 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9490 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9494 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9495 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9496 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9497 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9498 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9499 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9500 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9502 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9503 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9504 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9518 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9536 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9537 }
9538 switch (enmEvent2)
9539 {
9540 /** @todo consider which extra parameters would be helpful for each probe. */
9541 case DBGFEVENT_END: break;
9542 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9544 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9552 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9553 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9554 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9555 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9556 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9557 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9558 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9594 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9595 }
9596 }
9597
9598 /*
9599 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9600 * the DBGF call will do a full check).
9601 *
9602 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9603 * Note! If we have to events, we prioritize the first, i.e. the instruction
9604 * one, in order to avoid event nesting.
9605 */
9606 PVM pVM = pVCpu->CTX_SUFF(pVM);
9607 if ( enmEvent1 != DBGFEVENT_END
9608 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9609 {
9610 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9611 if (rcStrict != VINF_SUCCESS)
9612 return rcStrict;
9613 }
9614 else if ( enmEvent2 != DBGFEVENT_END
9615 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9616 {
9617 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9618 if (rcStrict != VINF_SUCCESS)
9619 return rcStrict;
9620 }
9621
9622 return VINF_SUCCESS;
9623}
9624
9625
9626/**
9627 * Single-stepping VM-exit filtering.
9628 *
9629 * This is preprocessing the VM-exits and deciding whether we've gotten far
9630 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9631 * handling is performed.
9632 *
9633 * @returns Strict VBox status code (i.e. informational status codes too).
9634 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9635 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9636 * out-of-sync. Make sure to update the required
9637 * fields before using them.
9638 * @param pVmxTransient Pointer to the VMX-transient structure.
9639 * @param uExitReason The VM-exit reason.
9640 * @param pDbgState The debug state.
9641 */
9642DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9643 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9644{
9645 /*
9646 * Expensive (saves context) generic dtrace VM-exit probe.
9647 */
9648 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9649 { /* more likely */ }
9650 else
9651 {
9652 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9653 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9654 AssertRC(rc);
9655 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9656 }
9657
9658 /*
9659 * Check for host NMI, just to get that out of the way.
9660 */
9661 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9662 { /* normally likely */ }
9663 else
9664 {
9665 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9666 AssertRCReturn(rc2, rc2);
9667 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9668 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9669 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9670 }
9671
9672 /*
9673 * Check for single stepping event if we're stepping.
9674 */
9675 if (pVCpu->hm.s.fSingleInstruction)
9676 {
9677 switch (uExitReason)
9678 {
9679 case VMX_EXIT_MTF:
9680 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9681
9682 /* Various events: */
9683 case VMX_EXIT_XCPT_OR_NMI:
9684 case VMX_EXIT_EXT_INT:
9685 case VMX_EXIT_TRIPLE_FAULT:
9686 case VMX_EXIT_INT_WINDOW:
9687 case VMX_EXIT_NMI_WINDOW:
9688 case VMX_EXIT_TASK_SWITCH:
9689 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9690 case VMX_EXIT_APIC_ACCESS:
9691 case VMX_EXIT_EPT_VIOLATION:
9692 case VMX_EXIT_EPT_MISCONFIG:
9693 case VMX_EXIT_PREEMPT_TIMER:
9694
9695 /* Instruction specific VM-exits: */
9696 case VMX_EXIT_CPUID:
9697 case VMX_EXIT_GETSEC:
9698 case VMX_EXIT_HLT:
9699 case VMX_EXIT_INVD:
9700 case VMX_EXIT_INVLPG:
9701 case VMX_EXIT_RDPMC:
9702 case VMX_EXIT_RDTSC:
9703 case VMX_EXIT_RSM:
9704 case VMX_EXIT_VMCALL:
9705 case VMX_EXIT_VMCLEAR:
9706 case VMX_EXIT_VMLAUNCH:
9707 case VMX_EXIT_VMPTRLD:
9708 case VMX_EXIT_VMPTRST:
9709 case VMX_EXIT_VMREAD:
9710 case VMX_EXIT_VMRESUME:
9711 case VMX_EXIT_VMWRITE:
9712 case VMX_EXIT_VMXOFF:
9713 case VMX_EXIT_VMXON:
9714 case VMX_EXIT_MOV_CRX:
9715 case VMX_EXIT_MOV_DRX:
9716 case VMX_EXIT_IO_INSTR:
9717 case VMX_EXIT_RDMSR:
9718 case VMX_EXIT_WRMSR:
9719 case VMX_EXIT_MWAIT:
9720 case VMX_EXIT_MONITOR:
9721 case VMX_EXIT_PAUSE:
9722 case VMX_EXIT_XDTR_ACCESS:
9723 case VMX_EXIT_TR_ACCESS:
9724 case VMX_EXIT_INVEPT:
9725 case VMX_EXIT_RDTSCP:
9726 case VMX_EXIT_INVVPID:
9727 case VMX_EXIT_WBINVD:
9728 case VMX_EXIT_XSETBV:
9729 case VMX_EXIT_RDRAND:
9730 case VMX_EXIT_INVPCID:
9731 case VMX_EXIT_VMFUNC:
9732 case VMX_EXIT_RDSEED:
9733 case VMX_EXIT_XSAVES:
9734 case VMX_EXIT_XRSTORS:
9735 {
9736 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9737 | CPUMCTX_EXTRN_CS);
9738 AssertRCReturn(rc, rc);
9739 if ( pMixedCtx->rip != pDbgState->uRipStart
9740 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9741 return VINF_EM_DBG_STEPPED;
9742 break;
9743 }
9744
9745 /* Errors and unexpected events: */
9746 case VMX_EXIT_INIT_SIGNAL:
9747 case VMX_EXIT_SIPI:
9748 case VMX_EXIT_IO_SMI:
9749 case VMX_EXIT_SMI:
9750 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9751 case VMX_EXIT_ERR_MSR_LOAD:
9752 case VMX_EXIT_ERR_MACHINE_CHECK:
9753 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9754 break;
9755
9756 default:
9757 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9758 break;
9759 }
9760 }
9761
9762 /*
9763 * Check for debugger event breakpoints and dtrace probes.
9764 */
9765 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9766 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9767 {
9768 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9769 if (rcStrict != VINF_SUCCESS)
9770 return rcStrict;
9771 }
9772
9773 /*
9774 * Normal processing.
9775 */
9776#ifdef HMVMX_USE_FUNCTION_TABLE
9777 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9778#else
9779 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9780#endif
9781}
9782
9783
9784/**
9785 * Single steps guest code using VT-x.
9786 *
9787 * @returns Strict VBox status code (i.e. informational status codes too).
9788 * @param pVCpu The cross context virtual CPU structure.
9789 * @param pCtx Pointer to the guest-CPU context.
9790 *
9791 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9792 */
9793static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9794{
9795 VMXTRANSIENT VmxTransient;
9796 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9797
9798 /* Set HMCPU indicators. */
9799 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9800 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9801 pVCpu->hm.s.fDebugWantRdTscExit = false;
9802 pVCpu->hm.s.fUsingDebugLoop = true;
9803
9804 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9805 VMXRUNDBGSTATE DbgState;
9806 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9807 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9808
9809 /*
9810 * The loop.
9811 */
9812 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9813 for (uint32_t cLoops = 0; ; cLoops++)
9814 {
9815 Assert(!HMR0SuspendPending());
9816 HMVMX_ASSERT_CPU_SAFE();
9817 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9818
9819 /*
9820 * Preparatory work for running guest code, this may force us to return
9821 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9822 */
9823 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9824 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9825 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9826 if (rcStrict != VINF_SUCCESS)
9827 break;
9828
9829 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9830 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9831
9832 /*
9833 * Now we can run the guest code.
9834 */
9835 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9836
9837 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9838
9839 /*
9840 * Restore any residual host-state and save any bits shared between host
9841 * and guest into the guest-CPU state. Re-enables interrupts!
9842 */
9843 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9844
9845 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9846 if (RT_SUCCESS(rcRun))
9847 { /* very likely */ }
9848 else
9849 {
9850 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9851 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9852 return rcRun;
9853 }
9854
9855 /* Profile the VM-exit. */
9856 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9857 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9858 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9859 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9860 HMVMX_START_EXIT_DISPATCH_PROF();
9861
9862 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9863
9864 /*
9865 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9866 */
9867 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9869 if (rcStrict != VINF_SUCCESS)
9870 break;
9871 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9872 {
9873 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9874 rcStrict = VINF_EM_RAW_INTERRUPT;
9875 break;
9876 }
9877
9878 /*
9879 * Stepping: Did the RIP change, if so, consider it a single step.
9880 * Otherwise, make sure one of the TFs gets set.
9881 */
9882 if (fStepping)
9883 {
9884 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9885 | CPUMCTX_EXTRN_CS);
9886 AssertRC(rc);
9887 if ( pCtx->rip != DbgState.uRipStart
9888 || pCtx->cs.Sel != DbgState.uCsStart)
9889 {
9890 rcStrict = VINF_EM_DBG_STEPPED;
9891 break;
9892 }
9893 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9894 }
9895
9896 /*
9897 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9898 */
9899 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9900 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9901 }
9902
9903 /*
9904 * Clear the X86_EFL_TF if necessary.
9905 */
9906 if (pVCpu->hm.s.fClearTrapFlag)
9907 {
9908 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9909 AssertRC(rc);
9910 pVCpu->hm.s.fClearTrapFlag = false;
9911 pCtx->eflags.Bits.u1TF = 0;
9912 }
9913 /** @todo there seems to be issues with the resume flag when the monitor trap
9914 * flag is pending without being used. Seen early in bios init when
9915 * accessing APIC page in protected mode. */
9916
9917 /*
9918 * Restore VM-exit control settings as we may not reenter this function the
9919 * next time around.
9920 */
9921 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9922
9923 /* Restore HMCPU indicators. */
9924 pVCpu->hm.s.fUsingDebugLoop = false;
9925 pVCpu->hm.s.fDebugWantRdTscExit = false;
9926 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9927
9928 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9929 return rcStrict;
9930}
9931
9932
9933/** @} */
9934
9935
9936/**
9937 * Checks if any expensive dtrace probes are enabled and we should go to the
9938 * debug loop.
9939 *
9940 * @returns true if we should use debug loop, false if not.
9941 */
9942static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9943{
9944 /* It's probably faster to OR the raw 32-bit counter variables together.
9945 Since the variables are in an array and the probes are next to one
9946 another (more or less), we have good locality. So, better read
9947 eight-nine cache lines ever time and only have one conditional, than
9948 128+ conditionals, right? */
9949 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9950 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9951 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9952 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9953 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9954 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9955 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9956 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9957 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9958 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9959 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9960 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9961 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9962 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9963 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9964 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9965 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9966 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9967 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9968 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9969 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9970 ) != 0
9971 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9972 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9973 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9974 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9975 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9976 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9977 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9978 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9979 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9980 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9981 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9982 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9983 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9984 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9985 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9986 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9987 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9988 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9989 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9990 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9991 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9992 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9993 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9994 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9995 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9996 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9997 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9998 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9999 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10000 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10001 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10002 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10003 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10004 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10005 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10006 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10007 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10008 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10009 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10010 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10011 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10012 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10013 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10014 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10015 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10016 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10017 ) != 0
10018 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10019 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10020 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10021 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10022 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10023 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10024 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10025 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10026 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10027 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10028 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10029 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10030 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10031 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10032 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10033 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10034 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10035 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10036 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10037 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10038 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10039 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10040 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10041 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10042 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10043 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10044 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10045 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10046 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10047 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10048 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10049 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10050 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10051 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10052 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10053 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10054 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10055 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10056 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10057 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10058 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10059 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10060 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10061 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10062 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10063 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10064 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10065 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10066 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10067 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10068 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10069 ) != 0;
10070}
10071
10072
10073/**
10074 * Runs the guest code using VT-x.
10075 *
10076 * @returns Strict VBox status code (i.e. informational status codes too).
10077 * @param pVCpu The cross context virtual CPU structure.
10078 * @param pCtx Pointer to the guest-CPU context.
10079 */
10080VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10081{
10082 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10083 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10084 HMVMX_ASSERT_PREEMPT_SAFE();
10085
10086 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10087
10088 VBOXSTRICTRC rcStrict;
10089 if ( !pVCpu->hm.s.fUseDebugLoop
10090 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10091 && !DBGFIsStepping(pVCpu)
10092 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10093 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10094 else
10095 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10096
10097 if (rcStrict == VERR_EM_INTERPRETER)
10098 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10099 else if (rcStrict == VINF_EM_RESET)
10100 rcStrict = VINF_EM_TRIPLE_FAULT;
10101
10102 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10103 if (RT_FAILURE(rc2))
10104 {
10105 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10106 rcStrict = rc2;
10107 }
10108 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10109 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10110 return rcStrict;
10111}
10112
10113
10114#ifndef HMVMX_USE_FUNCTION_TABLE
10115DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10116{
10117#ifdef DEBUG_ramshankar
10118#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10119 do { \
10120 if (a_fSave != 0) \
10121 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10122 VBOXSTRICTRC rcStrict = a_CallExpr; \
10123 if (a_fSave != 0) \
10124 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10125 return rcStrict; \
10126 } while (0)
10127#else
10128# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10129#endif
10130 switch (rcReason)
10131 {
10132 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10141 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10142 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10143 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10144 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10145 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10146 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10147 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10148 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10149 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10150 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10151 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10152 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10153 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10154 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10155 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10156 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10157 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10158 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10159 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10160 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10161 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10162 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10163 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10164 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10165 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10166
10167 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10168 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10169 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10170 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10171 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10172 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10173 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10174 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10175 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10176
10177 case VMX_EXIT_VMCLEAR:
10178 case VMX_EXIT_VMLAUNCH:
10179 case VMX_EXIT_VMPTRLD:
10180 case VMX_EXIT_VMPTRST:
10181 case VMX_EXIT_VMREAD:
10182 case VMX_EXIT_VMRESUME:
10183 case VMX_EXIT_VMWRITE:
10184 case VMX_EXIT_VMXOFF:
10185 case VMX_EXIT_VMXON:
10186 case VMX_EXIT_INVEPT:
10187 case VMX_EXIT_INVVPID:
10188 case VMX_EXIT_VMFUNC:
10189 case VMX_EXIT_XSAVES:
10190 case VMX_EXIT_XRSTORS:
10191 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10192
10193 case VMX_EXIT_ENCLS:
10194 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10195 case VMX_EXIT_PML_FULL:
10196 default:
10197 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10198 }
10199#undef VMEXIT_CALL_RET
10200}
10201#endif /* !HMVMX_USE_FUNCTION_TABLE */
10202
10203
10204#ifdef VBOX_STRICT
10205/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10206# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10207 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10208
10209# define HMVMX_ASSERT_PREEMPT_CPUID() \
10210 do { \
10211 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10212 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10213 } while (0)
10214
10215# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10216 do { \
10217 AssertPtr(pVCpu); \
10218 AssertPtr(pMixedCtx); \
10219 AssertPtr(pVmxTransient); \
10220 Assert(pVmxTransient->fVMEntryFailed == false); \
10221 Assert(ASMIntAreEnabled()); \
10222 HMVMX_ASSERT_PREEMPT_SAFE(); \
10223 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10224 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10225 HMVMX_ASSERT_PREEMPT_SAFE(); \
10226 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10227 HMVMX_ASSERT_PREEMPT_CPUID(); \
10228 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10229 } while (0)
10230
10231# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10232 do { \
10233 Log4Func(("\n")); \
10234 } while (0)
10235#else /* nonstrict builds: */
10236# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10237 do { \
10238 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10239 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10240 } while (0)
10241# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10242#endif
10243
10244
10245/**
10246 * Advances the guest RIP by the specified number of bytes.
10247 *
10248 * @param pVCpu The cross context virtual CPU structure.
10249 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10250 * out-of-sync. Make sure to update the required fields
10251 * before using them.
10252 * @param cbInstr Number of bytes to advance the RIP by.
10253 *
10254 * @remarks No-long-jump zone!!!
10255 */
10256DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10257{
10258 /* Advance the RIP. */
10259 pMixedCtx->rip += cbInstr;
10260 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10261
10262 /* Update interrupt inhibition. */
10263 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10264 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10265 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10266}
10267
10268
10269/**
10270 * Advances the guest RIP after reading it from the VMCS.
10271 *
10272 * @returns VBox status code, no informational status codes.
10273 * @param pVCpu The cross context virtual CPU structure.
10274 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10275 * out-of-sync. Make sure to update the required fields
10276 * before using them.
10277 * @param pVmxTransient Pointer to the VMX transient structure.
10278 *
10279 * @remarks No-long-jump zone!!!
10280 */
10281static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10282{
10283 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10284 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10285 | CPUMCTX_EXTRN_RFLAGS);
10286 AssertRCReturn(rc, rc);
10287
10288 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10289
10290 /*
10291 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10292 * pending debug exception field as it takes care of priority of events.
10293 *
10294 * See Intel spec. 32.2.1 "Debug Exceptions".
10295 */
10296 if ( !pVCpu->hm.s.fSingleInstruction
10297 && pMixedCtx->eflags.Bits.u1TF)
10298 {
10299 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10300 AssertRCReturn(rc, rc);
10301 }
10302
10303 return VINF_SUCCESS;
10304}
10305
10306
10307/**
10308 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10309 * and update error record fields accordingly.
10310 *
10311 * @return VMX_IGS_* return codes.
10312 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10313 * wrong with the guest state.
10314 *
10315 * @param pVCpu The cross context virtual CPU structure.
10316 * @param pCtx Pointer to the guest-CPU state.
10317 *
10318 * @remarks This function assumes our cache of the VMCS controls
10319 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10320 */
10321static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10322{
10323#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10324#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10325 uError = (err); \
10326 break; \
10327 } else do { } while (0)
10328
10329 int rc;
10330 PVM pVM = pVCpu->CTX_SUFF(pVM);
10331 uint32_t uError = VMX_IGS_ERROR;
10332 uint32_t u32Val;
10333 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10334
10335 do
10336 {
10337 /*
10338 * CR0.
10339 */
10340 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10341 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10342 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10343 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10344 if (fUnrestrictedGuest)
10345 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10346
10347 uint32_t u32GuestCr0;
10348 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10349 AssertRCBreak(rc);
10350 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10351 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10352 if ( !fUnrestrictedGuest
10353 && (u32GuestCr0 & X86_CR0_PG)
10354 && !(u32GuestCr0 & X86_CR0_PE))
10355 {
10356 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10357 }
10358
10359 /*
10360 * CR4.
10361 */
10362 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10363 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10364
10365 uint32_t u32GuestCr4;
10366 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10367 AssertRCBreak(rc);
10368 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10369 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10370
10371 /*
10372 * IA32_DEBUGCTL MSR.
10373 */
10374 uint64_t u64Val;
10375 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10376 AssertRCBreak(rc);
10377 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10378 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10379 {
10380 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10381 }
10382 uint64_t u64DebugCtlMsr = u64Val;
10383
10384#ifdef VBOX_STRICT
10385 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10386 AssertRCBreak(rc);
10387 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10388#endif
10389 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10390
10391 /*
10392 * RIP and RFLAGS.
10393 */
10394 uint32_t u32Eflags;
10395#if HC_ARCH_BITS == 64
10396 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10397 AssertRCBreak(rc);
10398 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10399 if ( !fLongModeGuest
10400 || !pCtx->cs.Attr.n.u1Long)
10401 {
10402 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10403 }
10404 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10405 * must be identical if the "IA-32e mode guest" VM-entry
10406 * control is 1 and CS.L is 1. No check applies if the
10407 * CPU supports 64 linear-address bits. */
10408
10409 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10410 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10411 AssertRCBreak(rc);
10412 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10413 VMX_IGS_RFLAGS_RESERVED);
10414 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10415 u32Eflags = u64Val;
10416#else
10417 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10418 AssertRCBreak(rc);
10419 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10420 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10421#endif
10422
10423 if ( fLongModeGuest
10424 || ( fUnrestrictedGuest
10425 && !(u32GuestCr0 & X86_CR0_PE)))
10426 {
10427 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10428 }
10429
10430 uint32_t u32EntryInfo;
10431 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10432 AssertRCBreak(rc);
10433 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10434 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10435 {
10436 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10437 }
10438
10439 /*
10440 * 64-bit checks.
10441 */
10442#if HC_ARCH_BITS == 64
10443 if (fLongModeGuest)
10444 {
10445 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10446 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10447 }
10448
10449 if ( !fLongModeGuest
10450 && (u32GuestCr4 & X86_CR4_PCIDE))
10451 {
10452 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10453 }
10454
10455 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10456 * 51:32 beyond the processor's physical-address width are 0. */
10457
10458 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10459 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10460 {
10461 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10462 }
10463
10464 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10465 AssertRCBreak(rc);
10466 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10467
10468 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10469 AssertRCBreak(rc);
10470 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10471#endif
10472
10473 /*
10474 * PERF_GLOBAL MSR.
10475 */
10476 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10477 {
10478 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10479 AssertRCBreak(rc);
10480 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10481 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10482 }
10483
10484 /*
10485 * PAT MSR.
10486 */
10487 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10488 {
10489 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10490 AssertRCBreak(rc);
10491 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10492 for (unsigned i = 0; i < 8; i++)
10493 {
10494 uint8_t u8Val = (u64Val & 0xff);
10495 if ( u8Val != 0 /* UC */
10496 && u8Val != 1 /* WC */
10497 && u8Val != 4 /* WT */
10498 && u8Val != 5 /* WP */
10499 && u8Val != 6 /* WB */
10500 && u8Val != 7 /* UC- */)
10501 {
10502 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10503 }
10504 u64Val >>= 8;
10505 }
10506 }
10507
10508 /*
10509 * EFER MSR.
10510 */
10511 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10512 {
10513 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10514 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10515 AssertRCBreak(rc);
10516 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10517 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10518 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10519 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10520 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10521 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10522 || !(u32GuestCr0 & X86_CR0_PG)
10523 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10524 VMX_IGS_EFER_LMA_LME_MISMATCH);
10525 }
10526
10527 /*
10528 * Segment registers.
10529 */
10530 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10531 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10532 if (!(u32Eflags & X86_EFL_VM))
10533 {
10534 /* CS */
10535 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10536 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10537 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10538 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10539 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10540 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10541 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10542 /* CS cannot be loaded with NULL in protected mode. */
10543 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10544 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10545 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10546 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10547 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10548 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10549 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10550 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10551 else
10552 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10553
10554 /* SS */
10555 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10556 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10557 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10558 if ( !(pCtx->cr0 & X86_CR0_PE)
10559 || pCtx->cs.Attr.n.u4Type == 3)
10560 {
10561 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10562 }
10563 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10564 {
10565 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10566 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10567 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10568 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10569 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10570 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10571 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10572 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10573 }
10574
10575 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10576 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10577 {
10578 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10579 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10580 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10581 || pCtx->ds.Attr.n.u4Type > 11
10582 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10583 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10584 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10585 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10586 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10587 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10588 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10589 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10590 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10591 }
10592 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10593 {
10594 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10595 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10596 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10597 || pCtx->es.Attr.n.u4Type > 11
10598 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10599 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10600 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10601 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10602 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10603 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10604 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10605 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10606 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10607 }
10608 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10609 {
10610 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10611 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10612 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10613 || pCtx->fs.Attr.n.u4Type > 11
10614 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10615 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10616 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10617 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10618 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10619 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10620 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10621 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10622 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10623 }
10624 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10625 {
10626 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10627 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10628 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10629 || pCtx->gs.Attr.n.u4Type > 11
10630 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10631 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10632 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10633 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10634 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10635 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10636 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10637 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10638 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10639 }
10640 /* 64-bit capable CPUs. */
10641#if HC_ARCH_BITS == 64
10642 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10643 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10644 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10645 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10646 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10647 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10648 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10649 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10650 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10651 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10652 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10653#endif
10654 }
10655 else
10656 {
10657 /* V86 mode checks. */
10658 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10659 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10660 {
10661 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10662 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10663 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10664 }
10665 else
10666 {
10667 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10668 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10669 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10670 }
10671
10672 /* CS */
10673 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10674 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10675 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10676 /* SS */
10677 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10678 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10679 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10680 /* DS */
10681 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10682 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10683 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10684 /* ES */
10685 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10686 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10687 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10688 /* FS */
10689 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10690 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10691 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10692 /* GS */
10693 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10694 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10695 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10696 /* 64-bit capable CPUs. */
10697#if HC_ARCH_BITS == 64
10698 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10699 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10700 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10701 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10702 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10703 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10704 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10705 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10706 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10707 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10708 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10709#endif
10710 }
10711
10712 /*
10713 * TR.
10714 */
10715 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10716 /* 64-bit capable CPUs. */
10717#if HC_ARCH_BITS == 64
10718 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10719#endif
10720 if (fLongModeGuest)
10721 {
10722 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10723 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10724 }
10725 else
10726 {
10727 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10728 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10729 VMX_IGS_TR_ATTR_TYPE_INVALID);
10730 }
10731 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10732 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10733 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10734 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10735 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10736 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10737 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10738 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10739
10740 /*
10741 * GDTR and IDTR.
10742 */
10743#if HC_ARCH_BITS == 64
10744 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10745 AssertRCBreak(rc);
10746 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10747
10748 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10749 AssertRCBreak(rc);
10750 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10751#endif
10752
10753 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10754 AssertRCBreak(rc);
10755 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10756
10757 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10758 AssertRCBreak(rc);
10759 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10760
10761 /*
10762 * Guest Non-Register State.
10763 */
10764 /* Activity State. */
10765 uint32_t u32ActivityState;
10766 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10767 AssertRCBreak(rc);
10768 HMVMX_CHECK_BREAK( !u32ActivityState
10769 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10770 VMX_IGS_ACTIVITY_STATE_INVALID);
10771 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10772 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10773 uint32_t u32IntrState;
10774 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10775 AssertRCBreak(rc);
10776 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10777 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10778 {
10779 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10780 }
10781
10782 /** @todo Activity state and injecting interrupts. Left as a todo since we
10783 * currently don't use activity states but ACTIVE. */
10784
10785 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10786 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10787
10788 /* Guest interruptibility-state. */
10789 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10790 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10791 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10792 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10793 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10794 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10795 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10796 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10797 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10798 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10799 {
10800 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10801 {
10802 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10803 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10804 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10805 }
10806 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10807 {
10808 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10809 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10810 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10811 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10812 }
10813 }
10814 /** @todo Assumes the processor is not in SMM. */
10815 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10816 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10817 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10818 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10819 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10820 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10821 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10822 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10823 {
10824 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10825 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10826 }
10827
10828 /* Pending debug exceptions. */
10829#if HC_ARCH_BITS == 64
10830 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10831 AssertRCBreak(rc);
10832 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10833 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10834 u32Val = u64Val; /* For pending debug exceptions checks below. */
10835#else
10836 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10837 AssertRCBreak(rc);
10838 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10839 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10840#endif
10841
10842 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10843 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10844 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10845 {
10846 if ( (u32Eflags & X86_EFL_TF)
10847 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10848 {
10849 /* Bit 14 is PendingDebug.BS. */
10850 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10851 }
10852 if ( !(u32Eflags & X86_EFL_TF)
10853 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10854 {
10855 /* Bit 14 is PendingDebug.BS. */
10856 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10857 }
10858 }
10859
10860 /* VMCS link pointer. */
10861 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10862 AssertRCBreak(rc);
10863 if (u64Val != UINT64_C(0xffffffffffffffff))
10864 {
10865 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10866 /** @todo Bits beyond the processor's physical-address width MBZ. */
10867 /** @todo 32-bit located in memory referenced by value of this field (as a
10868 * physical address) must contain the processor's VMCS revision ID. */
10869 /** @todo SMM checks. */
10870 }
10871
10872 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10873 * not using Nested Paging? */
10874 if ( pVM->hm.s.fNestedPaging
10875 && !fLongModeGuest
10876 && CPUMIsGuestInPAEModeEx(pCtx))
10877 {
10878 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10879 AssertRCBreak(rc);
10880 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10881
10882 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10883 AssertRCBreak(rc);
10884 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10885
10886 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10887 AssertRCBreak(rc);
10888 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10889
10890 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10891 AssertRCBreak(rc);
10892 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10893 }
10894
10895 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10896 if (uError == VMX_IGS_ERROR)
10897 uError = VMX_IGS_REASON_NOT_FOUND;
10898 } while (0);
10899
10900 pVCpu->hm.s.u32HMError = uError;
10901 return uError;
10902
10903#undef HMVMX_ERROR_BREAK
10904#undef HMVMX_CHECK_BREAK
10905}
10906
10907/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10908/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10909/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10910
10911/** @name VM-exit handlers.
10912 * @{
10913 */
10914
10915/**
10916 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10917 */
10918HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10919{
10920 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10921 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10922 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10923 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10924 return VINF_SUCCESS;
10925 return VINF_EM_RAW_INTERRUPT;
10926}
10927
10928
10929/**
10930 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10931 */
10932HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10933{
10934 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10935 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10936
10937 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10938 AssertRCReturn(rc, rc);
10939
10940 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10941 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10942 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10943 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10944
10945 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10946 {
10947 /*
10948 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10949 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10950 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10951 *
10952 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10953 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10954 */
10955 VMXDispatchHostNmi();
10956 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10957 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10958 return VINF_SUCCESS;
10959 }
10960
10961 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10962 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10963 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10964 { /* likely */ }
10965 else
10966 {
10967 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10968 rcStrictRc1 = VINF_SUCCESS;
10969 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10970 return rcStrictRc1;
10971 }
10972
10973 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10974 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10975 switch (uIntType)
10976 {
10977 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10978 Assert(uVector == X86_XCPT_DB);
10979 RT_FALL_THRU();
10980 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10981 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10982 RT_FALL_THRU();
10983 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10984 {
10985 /*
10986 * If there's any exception caused as a result of event injection, the resulting
10987 * secondary/final execption will be pending, we shall continue guest execution
10988 * after injecting the event. The page-fault case is complicated and we manually
10989 * handle any currently pending event in hmR0VmxExitXcptPF.
10990 */
10991 if (!pVCpu->hm.s.Event.fPending)
10992 { /* likely */ }
10993 else if (uVector != X86_XCPT_PF)
10994 {
10995 rc = VINF_SUCCESS;
10996 break;
10997 }
10998
10999 switch (uVector)
11000 {
11001 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11002 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11003 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11004 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11005 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11006 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11007
11008 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11009 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11010 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11011 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11012 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11013 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11014 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11015 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11016 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11017 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11018 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11019 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11020 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11021 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11022 default:
11023 {
11024 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11025 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11026 {
11027 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11028 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11029 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11030
11031 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11032 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11033 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11034 AssertRCReturn(rc, rc);
11035 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11036 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11037 0 /* GCPtrFaultAddress */);
11038 }
11039 else
11040 {
11041 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11042 pVCpu->hm.s.u32HMError = uVector;
11043 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11044 }
11045 break;
11046 }
11047 }
11048 break;
11049 }
11050
11051 default:
11052 {
11053 pVCpu->hm.s.u32HMError = uExitIntInfo;
11054 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11055 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11056 break;
11057 }
11058 }
11059 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11060 return rc;
11061}
11062
11063
11064/**
11065 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11066 */
11067HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11068{
11069 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11070
11071 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11072 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11073
11074 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11075 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11076 return VINF_SUCCESS;
11077}
11078
11079
11080/**
11081 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11082 */
11083HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11084{
11085 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11086 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11087 {
11088 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11089 HMVMX_RETURN_UNEXPECTED_EXIT();
11090 }
11091
11092 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11093
11094 /*
11095 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11096 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11097 */
11098 uint32_t fIntrState = 0;
11099 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11100 AssertRCReturn(rc, rc);
11101
11102 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11103 if ( fBlockSti
11104 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11105 {
11106 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11107 }
11108
11109 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11110 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11111
11112 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11113 return VINF_SUCCESS;
11114}
11115
11116
11117/**
11118 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11119 */
11120HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11121{
11122 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11123 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11124}
11125
11126
11127/**
11128 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11129 */
11130HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11131{
11132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11133 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11134}
11135
11136
11137/**
11138 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11139 */
11140HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11141{
11142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11143 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11144
11145 /*
11146 * Get the state we need and update the exit history entry.
11147 */
11148 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11149 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11150 | CPUMCTX_EXTRN_CS);
11151 AssertRCReturn(rc, rc);
11152
11153 VBOXSTRICTRC rcStrict;
11154 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11155 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11156 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11157 if (!pExitRec)
11158 {
11159 /*
11160 * Regular CPUID instruction execution.
11161 */
11162 PVM pVM = pVCpu->CTX_SUFF(pVM);
11163 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11164 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11165 {
11166 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11167 Assert(pVmxTransient->cbInstr == 2);
11168 }
11169 else
11170 {
11171 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11172 rcStrict = VERR_EM_INTERPRETER;
11173 }
11174 }
11175 else
11176 {
11177 /*
11178 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11179 */
11180 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11181 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11182 AssertRCReturn(rc2, rc2);
11183
11184 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11185 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11186
11187 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11189
11190 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11191 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11192 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11193 }
11194 return VBOXSTRICTRC_TODO(rcStrict);
11195}
11196
11197
11198/**
11199 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11200 */
11201HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11202{
11203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11204 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11205 AssertRCReturn(rc, rc);
11206
11207 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11208 return VINF_EM_RAW_EMULATE_INSTR;
11209
11210 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11211 HMVMX_RETURN_UNEXPECTED_EXIT();
11212}
11213
11214
11215/**
11216 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11217 */
11218HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11219{
11220 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11221 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11222 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11223 AssertRCReturn(rc, rc);
11224
11225 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11226 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11227 {
11228 /* If we get a spurious VM-exit when offsetting is enabled,
11229 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11230 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11231 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11232 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11233 | HM_CHANGED_GUEST_RFLAGS);
11234 }
11235 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11236 {
11237 rcStrict = VINF_SUCCESS;
11238 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11239 }
11240 return rcStrict;
11241}
11242
11243
11244/**
11245 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11246 */
11247HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11248{
11249 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11250 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11251 | CPUMCTX_EXTRN_TSC_AUX);
11252 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11253 AssertRCReturn(rc, rc);
11254
11255 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11256 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11257 {
11258 /* If we get a spurious VM-exit when offsetting is enabled,
11259 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11260 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11261 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11262 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11263 | HM_CHANGED_GUEST_RFLAGS);
11264 }
11265 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11266 {
11267 rcStrict = VINF_SUCCESS;
11268 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11269 }
11270 return rcStrict;
11271}
11272
11273
11274/**
11275 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11276 */
11277HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11278{
11279 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11280 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11281 | CPUMCTX_EXTRN_CR0
11282 | CPUMCTX_EXTRN_RFLAGS
11283 | CPUMCTX_EXTRN_SS);
11284 AssertRCReturn(rc, rc);
11285
11286 PVM pVM = pVCpu->CTX_SUFF(pVM);
11287 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11288 if (RT_LIKELY(rc == VINF_SUCCESS))
11289 {
11290 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11291 Assert(pVmxTransient->cbInstr == 2);
11292 }
11293 else
11294 {
11295 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11296 rc = VERR_EM_INTERPRETER;
11297 }
11298 return rc;
11299}
11300
11301
11302/**
11303 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11304 */
11305HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11306{
11307 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11308
11309 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11310 if (EMAreHypercallInstructionsEnabled(pVCpu))
11311 {
11312 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11313 | CPUMCTX_EXTRN_RFLAGS
11314 | CPUMCTX_EXTRN_CR0
11315 | CPUMCTX_EXTRN_SS
11316 | CPUMCTX_EXTRN_CS
11317 | CPUMCTX_EXTRN_EFER);
11318 AssertRCReturn(rc, rc);
11319
11320 /* Perform the hypercall. */
11321 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11322 if (rcStrict == VINF_SUCCESS)
11323 {
11324 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11325 AssertRCReturn(rc, rc);
11326 }
11327 else
11328 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11329 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11330 || RT_FAILURE(rcStrict));
11331
11332 /* If the hypercall changes anything other than guest's general-purpose registers,
11333 we would need to reload the guest changed bits here before VM-entry. */
11334 }
11335 else
11336 Log4Func(("Hypercalls not enabled\n"));
11337
11338 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11339 if (RT_FAILURE(rcStrict))
11340 {
11341 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11342 rcStrict = VINF_SUCCESS;
11343 }
11344
11345 return rcStrict;
11346}
11347
11348
11349/**
11350 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11351 */
11352HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11353{
11354 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11355 PVM pVM = pVCpu->CTX_SUFF(pVM);
11356 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11357
11358 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11359 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11360 AssertRCReturn(rc, rc);
11361
11362 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11363 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11364 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11365 else
11366 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11367 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11368 return rcStrict;
11369}
11370
11371
11372/**
11373 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11374 */
11375HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11376{
11377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11378 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11379 | CPUMCTX_EXTRN_RFLAGS
11380 | CPUMCTX_EXTRN_SS);
11381 AssertRCReturn(rc, rc);
11382
11383 PVM pVM = pVCpu->CTX_SUFF(pVM);
11384 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11385 if (RT_LIKELY(rc == VINF_SUCCESS))
11386 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11387 else
11388 {
11389 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11390 rc = VERR_EM_INTERPRETER;
11391 }
11392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11393 return rc;
11394}
11395
11396
11397/**
11398 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11399 */
11400HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11401{
11402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11403 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11404 | CPUMCTX_EXTRN_RFLAGS
11405 | CPUMCTX_EXTRN_SS);
11406 AssertRCReturn(rc, rc);
11407
11408 PVM pVM = pVCpu->CTX_SUFF(pVM);
11409 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11410 rc = VBOXSTRICTRC_VAL(rc2);
11411 if (RT_LIKELY( rc == VINF_SUCCESS
11412 || rc == VINF_EM_HALT))
11413 {
11414 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11415 AssertRCReturn(rc3, rc3);
11416
11417 if ( rc == VINF_EM_HALT
11418 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11419 rc = VINF_SUCCESS;
11420 }
11421 else
11422 {
11423 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11424 rc = VERR_EM_INTERPRETER;
11425 }
11426 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11427 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11428 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11429 return rc;
11430}
11431
11432
11433/**
11434 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11435 */
11436HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11437{
11438 /*
11439 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11440 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11441 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11442 * VMX root operation. If we get here, something funny is going on.
11443 *
11444 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11445 */
11446 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11447 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11448 HMVMX_RETURN_UNEXPECTED_EXIT();
11449}
11450
11451
11452/**
11453 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11454 */
11455HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11456{
11457 /*
11458 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11459 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11460 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11461 * an SMI. If we get here, something funny is going on.
11462 *
11463 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11464 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11465 */
11466 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11467 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11468 HMVMX_RETURN_UNEXPECTED_EXIT();
11469}
11470
11471
11472/**
11473 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11474 */
11475HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11476{
11477 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11479 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11480 HMVMX_RETURN_UNEXPECTED_EXIT();
11481}
11482
11483
11484/**
11485 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11486 */
11487HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11488{
11489 /*
11490 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11491 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11492 * See Intel spec. 25.3 "Other Causes of VM-exits".
11493 */
11494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11495 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11496 HMVMX_RETURN_UNEXPECTED_EXIT();
11497}
11498
11499
11500/**
11501 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11502 * VM-exit.
11503 */
11504HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11505{
11506 /*
11507 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11508 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11509 *
11510 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11511 * See Intel spec. "23.8 Restrictions on VMX operation".
11512 */
11513 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11514 return VINF_SUCCESS;
11515}
11516
11517
11518/**
11519 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11520 * VM-exit.
11521 */
11522HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11523{
11524 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11525 return VINF_EM_RESET;
11526}
11527
11528
11529/**
11530 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11531 */
11532HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11533{
11534 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11535 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11536
11537 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11538 AssertRCReturn(rc, rc);
11539
11540 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11541 rc = VINF_SUCCESS;
11542 else
11543 rc = VINF_EM_HALT;
11544
11545 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11546 if (rc != VINF_SUCCESS)
11547 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11548 return rc;
11549}
11550
11551
11552/**
11553 * VM-exit handler for instructions that result in a \#UD exception delivered to
11554 * the guest.
11555 */
11556HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11557{
11558 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11559 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11560 return VINF_SUCCESS;
11561}
11562
11563
11564/**
11565 * VM-exit handler for expiry of the VMX preemption timer.
11566 */
11567HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11568{
11569 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11570
11571 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11572 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11573
11574 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11575 PVM pVM = pVCpu->CTX_SUFF(pVM);
11576 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11578 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11579}
11580
11581
11582/**
11583 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11584 */
11585HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11586{
11587 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11588
11589 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11590 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11591 | CPUMCTX_EXTRN_CR4);
11592 AssertRCReturn(rc, rc);
11593
11594 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11595 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11596 : HM_CHANGED_XCPT_RAISED_MASK);
11597
11598 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11599
11600 return rcStrict;
11601}
11602
11603
11604/**
11605 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11606 */
11607HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11608{
11609 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11610 /** @todo Use VM-exit instruction information. */
11611 return VERR_EM_INTERPRETER;
11612}
11613
11614
11615/**
11616 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11617 * Error VM-exit.
11618 */
11619HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11620{
11621 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11622 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11623 AssertRCReturn(rc, rc);
11624
11625 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11626 NOREF(uInvalidReason);
11627
11628#ifdef VBOX_STRICT
11629 uint32_t fIntrState;
11630 RTHCUINTREG uHCReg;
11631 uint64_t u64Val;
11632 uint32_t u32Val;
11633
11634 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11635 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11636 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11637 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11638 AssertRCReturn(rc, rc);
11639
11640 Log4(("uInvalidReason %u\n", uInvalidReason));
11641 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11642 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11643 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11644 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11645
11646 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11647 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11648 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11649 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11650 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11651 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11652 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11653 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11654 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11655 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11656 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11657 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11658
11659 hmR0DumpRegs(pVCpu, pMixedCtx);
11660#else
11661 NOREF(pVmxTransient);
11662#endif
11663
11664 return VERR_VMX_INVALID_GUEST_STATE;
11665}
11666
11667
11668/**
11669 * VM-exit handler for VM-entry failure due to an MSR-load
11670 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11671 */
11672HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11673{
11674 NOREF(pVmxTransient);
11675 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11676 HMVMX_RETURN_UNEXPECTED_EXIT();
11677}
11678
11679
11680/**
11681 * VM-exit handler for VM-entry failure due to a machine-check event
11682 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11683 */
11684HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11685{
11686 NOREF(pVmxTransient);
11687 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11688 HMVMX_RETURN_UNEXPECTED_EXIT();
11689}
11690
11691
11692/**
11693 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11694 * theory.
11695 */
11696HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11697{
11698 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11699 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11700 return VERR_VMX_UNDEFINED_EXIT_CODE;
11701}
11702
11703
11704/**
11705 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11706 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11707 * Conditional VM-exit.
11708 */
11709HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11710{
11711 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11712
11713 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11714 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11715 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11716 return VERR_EM_INTERPRETER;
11717 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11718 HMVMX_RETURN_UNEXPECTED_EXIT();
11719}
11720
11721
11722/**
11723 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11724 */
11725HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11726{
11727 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11728
11729 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11730 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11731 return VERR_EM_INTERPRETER;
11732 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11733 HMVMX_RETURN_UNEXPECTED_EXIT();
11734}
11735
11736
11737/**
11738 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11739 */
11740HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11741{
11742 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11743
11744 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11745 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11746 | CPUMCTX_EXTRN_RFLAGS
11747 | CPUMCTX_EXTRN_SS);
11748 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11749 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11750 AssertRCReturn(rc, rc);
11751 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11752
11753#ifdef VBOX_STRICT
11754 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11755 {
11756 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11757 && pMixedCtx->ecx != MSR_K6_EFER)
11758 {
11759 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11760 pMixedCtx->ecx));
11761 HMVMX_RETURN_UNEXPECTED_EXIT();
11762 }
11763 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11764 {
11765 VMXMSREXITREAD enmRead;
11766 VMXMSREXITWRITE enmWrite;
11767 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11768 AssertRCReturn(rc2, rc2);
11769 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11770 {
11771 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11772 HMVMX_RETURN_UNEXPECTED_EXIT();
11773 }
11774 }
11775 }
11776#endif
11777
11778 PVM pVM = pVCpu->CTX_SUFF(pVM);
11779 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11780 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11781 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11782 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11783 if (RT_SUCCESS(rc))
11784 {
11785 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11786 Assert(pVmxTransient->cbInstr == 2);
11787 }
11788 return rc;
11789}
11790
11791
11792/**
11793 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11794 */
11795HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11796{
11797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11798 PVM pVM = pVCpu->CTX_SUFF(pVM);
11799 int rc = VINF_SUCCESS;
11800
11801 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11802 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11803 | CPUMCTX_EXTRN_RFLAGS
11804 | CPUMCTX_EXTRN_SS);
11805 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11806 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11807 AssertRCReturn(rc, rc);
11808 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11809
11810 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11811 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11812 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11813
11814 if (RT_SUCCESS(rc))
11815 {
11816 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11817
11818 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11819 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11820 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11821 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11822 {
11823 /*
11824 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11825 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11826 * EMInterpretWrmsr() changes it.
11827 */
11828 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11829 }
11830 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11831 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11832 else if (pMixedCtx->ecx == MSR_K6_EFER)
11833 {
11834 /*
11835 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11836 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11837 * the other bits as well, SCE and NXE. See @bugref{7368}.
11838 */
11839 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11840 | HM_CHANGED_VMX_ENTRY_CTLS
11841 | HM_CHANGED_VMX_EXIT_CTLS);
11842 }
11843
11844 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11845 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11846 {
11847 switch (pMixedCtx->ecx)
11848 {
11849 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11850 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11851 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11852 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11853 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11854 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11855 default:
11856 {
11857 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11858 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11859 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11860 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11861 break;
11862 }
11863 }
11864 }
11865#ifdef VBOX_STRICT
11866 else
11867 {
11868 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11869 switch (pMixedCtx->ecx)
11870 {
11871 case MSR_IA32_SYSENTER_CS:
11872 case MSR_IA32_SYSENTER_EIP:
11873 case MSR_IA32_SYSENTER_ESP:
11874 case MSR_K8_FS_BASE:
11875 case MSR_K8_GS_BASE:
11876 {
11877 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11878 HMVMX_RETURN_UNEXPECTED_EXIT();
11879 }
11880
11881 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11882 default:
11883 {
11884 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11885 {
11886 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11887 if (pMixedCtx->ecx != MSR_K6_EFER)
11888 {
11889 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11890 pMixedCtx->ecx));
11891 HMVMX_RETURN_UNEXPECTED_EXIT();
11892 }
11893 }
11894
11895 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11896 {
11897 VMXMSREXITREAD enmRead;
11898 VMXMSREXITWRITE enmWrite;
11899 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11900 AssertRCReturn(rc2, rc2);
11901 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11902 {
11903 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11904 HMVMX_RETURN_UNEXPECTED_EXIT();
11905 }
11906 }
11907 break;
11908 }
11909 }
11910 }
11911#endif /* VBOX_STRICT */
11912 }
11913 return rc;
11914}
11915
11916
11917/**
11918 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11919 */
11920HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11921{
11922 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11923 /** @todo The guest has likely hit a contended spinlock. We might want to
11924 * poke a schedule different guest VCPU. */
11925 return VINF_EM_RAW_INTERRUPT;
11926}
11927
11928
11929/**
11930 * VM-exit handler for when the TPR value is lowered below the specified
11931 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11932 */
11933HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11934{
11935 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11936 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11937
11938 /*
11939 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11940 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11941 */
11942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11943 return VINF_SUCCESS;
11944}
11945
11946
11947/**
11948 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11949 * VM-exit.
11950 *
11951 * @retval VINF_SUCCESS when guest execution can continue.
11952 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11953 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11954 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11955 * interpreter.
11956 */
11957HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11958{
11959 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11960 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11961
11962 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11963 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11964 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11965 AssertRCReturn(rc, rc);
11966
11967 VBOXSTRICTRC rcStrict;
11968 PVM pVM = pVCpu->CTX_SUFF(pVM);
11969 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11970 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11971 switch (uAccessType)
11972 {
11973 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11974 {
11975 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11976 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11977 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11978 AssertMsg( rcStrict == VINF_SUCCESS
11979 || rcStrict == VINF_IEM_RAISED_XCPT
11980 || rcStrict == VINF_PGM_CHANGE_MODE
11981 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11982
11983 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11984 {
11985 case 0:
11986 {
11987 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11989 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11990 break;
11991 }
11992
11993 case 2:
11994 {
11995 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11996 /* Nothing to do here, CR2 it's not part of the VMCS. */
11997 break;
11998 }
11999
12000 case 3:
12001 {
12002 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12004 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12005 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12006 break;
12007 }
12008
12009 case 4:
12010 {
12011 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12012 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12013 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12014 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12015 break;
12016 }
12017
12018 case 8:
12019 {
12020 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12021 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12022 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12023 break;
12024 }
12025 default:
12026 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12027 break;
12028 }
12029 break;
12030 }
12031
12032 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12033 {
12034 Assert( !pVM->hm.s.fNestedPaging
12035 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12036 || pVCpu->hm.s.fUsingDebugLoop
12037 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12038 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12039 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12040 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12041
12042 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12043 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12044 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12045 AssertMsg( rcStrict == VINF_SUCCESS
12046 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12047#ifdef VBOX_WITH_STATISTICS
12048 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12049 {
12050 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12051 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12052 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12053 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12054 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12055 }
12056#endif
12057 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12058 VBOXSTRICTRC_VAL(rcStrict)));
12059 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12060 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12061 break;
12062 }
12063
12064 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12065 {
12066 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12067 AssertMsg( rcStrict == VINF_SUCCESS
12068 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12069
12070 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12072 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12073 break;
12074 }
12075
12076 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12077 {
12078 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12079 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12080 AssertMsg( rcStrict == VINF_SUCCESS
12081 || rcStrict == VINF_IEM_RAISED_XCPT
12082 || rcStrict == VINF_PGM_CHANGE_MODE,
12083 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12084
12085 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12087 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12088 break;
12089 }
12090
12091 default:
12092 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12093 VERR_VMX_UNEXPECTED_EXCEPTION);
12094 }
12095
12096 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12097 : HM_CHANGED_XCPT_RAISED_MASK);
12098 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12099 NOREF(pVM);
12100 return rcStrict;
12101}
12102
12103
12104/**
12105 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12106 * VM-exit.
12107 */
12108HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12109{
12110 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12111 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12112 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12113
12114 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12115 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12116 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12117 | CPUMCTX_EXTRN_SREG_MASK
12118 | CPUMCTX_EXTRN_EFER);
12119 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12120 AssertRCReturn(rc, rc);
12121
12122 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12123 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12124 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12125 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12126 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12127 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12128 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12129 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12130 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12131
12132 /*
12133 * Update exit history to see if this exit can be optimized.
12134 */
12135 VBOXSTRICTRC rcStrict;
12136 PCEMEXITREC pExitRec = NULL;
12137 if ( !fGstStepping
12138 && !fDbgStepping)
12139 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12140 !fIOString
12141 ? !fIOWrite
12142 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12143 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12144 : !fIOWrite
12145 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12146 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12147 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12148 if (!pExitRec)
12149 {
12150 /* I/O operation lookup arrays. */
12151 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12152 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12153 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12154 uint32_t const cbInstr = pVmxTransient->cbInstr;
12155 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12156 PVM pVM = pVCpu->CTX_SUFF(pVM);
12157 if (fIOString)
12158 {
12159 /*
12160 * INS/OUTS - I/O String instruction.
12161 *
12162 * Use instruction-information if available, otherwise fall back on
12163 * interpreting the instruction.
12164 */
12165 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12166 fIOWrite ? 'w' : 'r'));
12167 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12168 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12169 {
12170 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12171 AssertRCReturn(rc2, rc2);
12172 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12173 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12174 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12175 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12176 if (fIOWrite)
12177 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12178 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12179 else
12180 {
12181 /*
12182 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12183 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12184 * See Intel Instruction spec. for "INS".
12185 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12186 */
12187 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12188 }
12189 }
12190 else
12191 rcStrict = IEMExecOne(pVCpu);
12192
12193 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12194 fUpdateRipAlready = true;
12195 }
12196 else
12197 {
12198 /*
12199 * IN/OUT - I/O instruction.
12200 */
12201 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12202 fIOWrite ? 'w' : 'r'));
12203 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12204 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12205 if (fIOWrite)
12206 {
12207 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12208 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12209 }
12210 else
12211 {
12212 uint32_t u32Result = 0;
12213 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12214 if (IOM_SUCCESS(rcStrict))
12215 {
12216 /* Save result of I/O IN instr. in AL/AX/EAX. */
12217 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12218 }
12219 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12220 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12222 }
12223 }
12224
12225 if (IOM_SUCCESS(rcStrict))
12226 {
12227 if (!fUpdateRipAlready)
12228 {
12229 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12230 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12231 }
12232
12233 /*
12234 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12235 * while booting Fedora 17 64-bit guest.
12236 *
12237 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12238 */
12239 if (fIOString)
12240 {
12241 /** @todo Single-step for INS/OUTS with REP prefix? */
12242 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12243 }
12244 else if ( !fDbgStepping
12245 && fGstStepping)
12246 {
12247 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12248 AssertRCReturn(rc, rc);
12249 }
12250
12251 /*
12252 * If any I/O breakpoints are armed, we need to check if one triggered
12253 * and take appropriate action.
12254 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12255 */
12256 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12257 AssertRCReturn(rc, rc);
12258
12259 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12260 * execution engines about whether hyper BPs and such are pending. */
12261 uint32_t const uDr7 = pMixedCtx->dr[7];
12262 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12263 && X86_DR7_ANY_RW_IO(uDr7)
12264 && (pMixedCtx->cr4 & X86_CR4_DE))
12265 || DBGFBpIsHwIoArmed(pVM)))
12266 {
12267 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12268
12269 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12270 VMMRZCallRing3Disable(pVCpu);
12271 HM_DISABLE_PREEMPT();
12272
12273 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12274
12275 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12276 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12277 {
12278 /* Raise #DB. */
12279 if (fIsGuestDbgActive)
12280 ASMSetDR6(pMixedCtx->dr[6]);
12281 if (pMixedCtx->dr[7] != uDr7)
12282 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12283
12284 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12285 }
12286 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12287 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12288 else if ( rcStrict2 != VINF_SUCCESS
12289 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12290 rcStrict = rcStrict2;
12291 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12292
12293 HM_RESTORE_PREEMPT();
12294 VMMRZCallRing3Enable(pVCpu);
12295 }
12296 }
12297
12298#ifdef VBOX_STRICT
12299 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12300 Assert(!fIOWrite);
12301 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12302 Assert(fIOWrite);
12303 else
12304 {
12305# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12306 * statuses, that the VMM device and some others may return. See
12307 * IOM_SUCCESS() for guidance. */
12308 AssertMsg( RT_FAILURE(rcStrict)
12309 || rcStrict == VINF_SUCCESS
12310 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12311 || rcStrict == VINF_EM_DBG_BREAKPOINT
12312 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12313 || rcStrict == VINF_EM_RAW_TO_R3
12314 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12315# endif
12316 }
12317#endif
12318 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12319 }
12320 else
12321 {
12322 /*
12323 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12324 */
12325 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12326 AssertRCReturn(rc2, rc2);
12327 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12328 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12329 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12330 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12331 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12332 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12333
12334 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12335 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12336
12337 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12338 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12339 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12340 }
12341 return rcStrict;
12342}
12343
12344
12345/**
12346 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12347 * VM-exit.
12348 */
12349HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12350{
12351 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12352
12353 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12354 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12355 AssertRCReturn(rc, rc);
12356 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12357 {
12358 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12359 AssertRCReturn(rc, rc);
12360 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12361 {
12362 uint32_t uErrCode;
12363 RTGCUINTPTR GCPtrFaultAddress;
12364 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12365 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12366 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12367 if (fErrorCodeValid)
12368 {
12369 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12370 AssertRCReturn(rc, rc);
12371 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12372 }
12373 else
12374 uErrCode = 0;
12375
12376 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12377 && uVector == X86_XCPT_PF)
12378 GCPtrFaultAddress = pMixedCtx->cr2;
12379 else
12380 GCPtrFaultAddress = 0;
12381
12382 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12383 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12384
12385 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12387 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12388 }
12389 }
12390
12391 /* Fall back to the interpreter to emulate the task-switch. */
12392 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12393 return VERR_EM_INTERPRETER;
12394}
12395
12396
12397/**
12398 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12399 */
12400HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12401{
12402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12403 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12404 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12405 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12406 AssertRCReturn(rc, rc);
12407 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12408 return VINF_EM_DBG_STEPPED;
12409}
12410
12411
12412/**
12413 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12414 */
12415HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12416{
12417 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12418
12419 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12420
12421 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12422 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12423 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12424 {
12425 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12426 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12427 {
12428 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12429 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12430 }
12431 }
12432 else
12433 {
12434 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12435 rcStrict1 = VINF_SUCCESS;
12436 return rcStrict1;
12437 }
12438
12439 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12440 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12441 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12442 AssertRCReturn(rc, rc);
12443
12444 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12445 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12446 VBOXSTRICTRC rcStrict2;
12447 switch (uAccessType)
12448 {
12449 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12450 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12451 {
12452 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12453 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12454 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12455
12456 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12457 GCPhys &= PAGE_BASE_GC_MASK;
12458 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12459 PVM pVM = pVCpu->CTX_SUFF(pVM);
12460 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12461 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12462
12463 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12464 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12465 CPUMCTX2CORE(pMixedCtx), GCPhys);
12466 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12467 if ( rcStrict2 == VINF_SUCCESS
12468 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12469 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12470 {
12471 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12472 | HM_CHANGED_GUEST_RSP
12473 | HM_CHANGED_GUEST_RFLAGS
12474 | HM_CHANGED_GUEST_APIC_TPR);
12475 rcStrict2 = VINF_SUCCESS;
12476 }
12477 break;
12478 }
12479
12480 default:
12481 Log4Func(("uAccessType=%#x\n", uAccessType));
12482 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12483 break;
12484 }
12485
12486 if (rcStrict2 != VINF_SUCCESS)
12487 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12488 return rcStrict2;
12489}
12490
12491
12492/**
12493 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12494 * VM-exit.
12495 */
12496HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12497{
12498 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12499
12500 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12501 if (pVmxTransient->fWasGuestDebugStateActive)
12502 {
12503 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12504 HMVMX_RETURN_UNEXPECTED_EXIT();
12505 }
12506
12507 if ( !pVCpu->hm.s.fSingleInstruction
12508 && !pVmxTransient->fWasHyperDebugStateActive)
12509 {
12510 Assert(!DBGFIsStepping(pVCpu));
12511 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12512
12513 /* Don't intercept MOV DRx any more. */
12514 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12515 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12516 AssertRCReturn(rc, rc);
12517
12518 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12519 VMMRZCallRing3Disable(pVCpu);
12520 HM_DISABLE_PREEMPT();
12521
12522 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12523 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12524 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12525
12526 HM_RESTORE_PREEMPT();
12527 VMMRZCallRing3Enable(pVCpu);
12528
12529#ifdef VBOX_WITH_STATISTICS
12530 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12531 AssertRCReturn(rc, rc);
12532 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12533 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12534 else
12535 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12536#endif
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12538 return VINF_SUCCESS;
12539 }
12540
12541 /*
12542 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12543 * Update the segment registers and DR7 from the CPU.
12544 */
12545 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12546 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12547 | CPUMCTX_EXTRN_DR7);
12548 AssertRCReturn(rc, rc);
12549 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12550
12551 PVM pVM = pVCpu->CTX_SUFF(pVM);
12552 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12553 {
12554 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12555 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12556 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12557 if (RT_SUCCESS(rc))
12558 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12559 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12560 }
12561 else
12562 {
12563 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12564 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12565 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12566 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12567 }
12568
12569 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12570 if (RT_SUCCESS(rc))
12571 {
12572 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12573 AssertRCReturn(rc2, rc2);
12574 return VINF_SUCCESS;
12575 }
12576 return rc;
12577}
12578
12579
12580/**
12581 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12582 * Conditional VM-exit.
12583 */
12584HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12585{
12586 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12587 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12588
12589 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12590 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12591 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12592 {
12593 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12594 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12595 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12596 {
12597 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12598 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12599 }
12600 }
12601 else
12602 {
12603 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12604 rcStrict1 = VINF_SUCCESS;
12605 return rcStrict1;
12606 }
12607
12608 /*
12609 * Get sufficent state and update the exit history entry.
12610 */
12611 RTGCPHYS GCPhys;
12612 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12613 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12614 AssertRCReturn(rc, rc);
12615
12616 VBOXSTRICTRC rcStrict;
12617 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12618 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12619 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12620 if (!pExitRec)
12621 {
12622 /*
12623 * If we succeed, resume guest execution.
12624 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12625 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12626 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12627 * weird case. See @bugref{6043}.
12628 */
12629 PVM pVM = pVCpu->CTX_SUFF(pVM);
12630 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12631 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12632 if ( rcStrict == VINF_SUCCESS
12633 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12634 || rcStrict == VERR_PAGE_NOT_PRESENT)
12635 {
12636 /* Successfully handled MMIO operation. */
12637 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12638 | HM_CHANGED_GUEST_RSP
12639 | HM_CHANGED_GUEST_RFLAGS
12640 | HM_CHANGED_GUEST_APIC_TPR);
12641 rcStrict = VINF_SUCCESS;
12642 }
12643 }
12644 else
12645 {
12646 /*
12647 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12648 */
12649 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12650 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12651 AssertRCReturn(rc2, rc2);
12652
12653 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12654 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12655
12656 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12657 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12658
12659 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12660 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12661 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12662 }
12663 return VBOXSTRICTRC_TODO(rcStrict);
12664}
12665
12666
12667/**
12668 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12669 * VM-exit.
12670 */
12671HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12672{
12673 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12674 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12675
12676 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12677 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12678 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12679 {
12680 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12681 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12682 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12683 }
12684 else
12685 {
12686 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12687 rcStrict1 = VINF_SUCCESS;
12688 return rcStrict1;
12689 }
12690
12691 RTGCPHYS GCPhys;
12692 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12693 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12694 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12695 AssertRCReturn(rc, rc);
12696
12697 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12698 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12699
12700 RTGCUINT uErrorCode = 0;
12701 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12702 uErrorCode |= X86_TRAP_PF_ID;
12703 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12704 uErrorCode |= X86_TRAP_PF_RW;
12705 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12706 uErrorCode |= X86_TRAP_PF_P;
12707
12708 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12709
12710 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12711 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12712
12713 /* Handle the pagefault trap for the nested shadow table. */
12714 PVM pVM = pVCpu->CTX_SUFF(pVM);
12715 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12716 TRPMResetTrap(pVCpu);
12717
12718 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12719 if ( rcStrict2 == VINF_SUCCESS
12720 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12721 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12722 {
12723 /* Successfully synced our nested page tables. */
12724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12725 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12726 | HM_CHANGED_GUEST_RSP
12727 | HM_CHANGED_GUEST_RFLAGS);
12728 return VINF_SUCCESS;
12729 }
12730
12731 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12732 return rcStrict2;
12733}
12734
12735/** @} */
12736
12737/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12738/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12739/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12740
12741/** @name VM-exit exception handlers.
12742 * @{
12743 */
12744
12745/**
12746 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12747 */
12748static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12749{
12750 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12751 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12752
12753 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12754 AssertRCReturn(rc, rc);
12755
12756 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12757 {
12758 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12759 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12760
12761 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12762 * provides VM-exit instruction length. If this causes problem later,
12763 * disassemble the instruction like it's done on AMD-V. */
12764 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12765 AssertRCReturn(rc2, rc2);
12766 return rc;
12767 }
12768
12769 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12770 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12771 return rc;
12772}
12773
12774
12775/**
12776 * VM-exit exception handler for \#BP (Breakpoint exception).
12777 */
12778static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12779{
12780 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12781 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12782
12783 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12784 AssertRCReturn(rc, rc);
12785
12786 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12787 if (rc == VINF_EM_RAW_GUEST_TRAP)
12788 {
12789 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12790 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12791 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12792 AssertRCReturn(rc, rc);
12793
12794 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12795 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12796 }
12797
12798 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12799 return rc;
12800}
12801
12802
12803/**
12804 * VM-exit exception handler for \#AC (alignment check exception).
12805 */
12806static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12807{
12808 RT_NOREF_PV(pMixedCtx);
12809 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12810
12811 /*
12812 * Re-inject it. We'll detect any nesting before getting here.
12813 */
12814 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12815 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12816 AssertRCReturn(rc, rc);
12817 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12818
12819 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12820 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12821 return VINF_SUCCESS;
12822}
12823
12824
12825/**
12826 * VM-exit exception handler for \#DB (Debug exception).
12827 */
12828static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12829{
12830 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12831 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12832
12833 /*
12834 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12835 * for processing.
12836 */
12837 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12838
12839 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12840 uint64_t uDR6 = X86_DR6_INIT_VAL;
12841 uDR6 |= ( pVmxTransient->uExitQualification
12842 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12843
12844 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12845 Log6Func(("rc=%Rrc\n", rc));
12846 if (rc == VINF_EM_RAW_GUEST_TRAP)
12847 {
12848 /*
12849 * The exception was for the guest. Update DR6, DR7.GD and
12850 * IA32_DEBUGCTL.LBR before forwarding it.
12851 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12852 */
12853 VMMRZCallRing3Disable(pVCpu);
12854 HM_DISABLE_PREEMPT();
12855
12856 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12857 pMixedCtx->dr[6] |= uDR6;
12858 if (CPUMIsGuestDebugStateActive(pVCpu))
12859 ASMSetDR6(pMixedCtx->dr[6]);
12860
12861 HM_RESTORE_PREEMPT();
12862 VMMRZCallRing3Enable(pVCpu);
12863
12864 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12865 AssertRCReturn(rc, rc);
12866
12867 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12868 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12869
12870 /* Paranoia. */
12871 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12872 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12873
12874 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12875 AssertRCReturn(rc, rc);
12876
12877 /*
12878 * Raise #DB in the guest.
12879 *
12880 * It is important to reflect exactly what the VM-exit gave us (preserving the
12881 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12882 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12883 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12884 *
12885 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12886 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12887 */
12888 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12889 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12890 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12891 AssertRCReturn(rc, rc);
12892 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12893 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12894 return VINF_SUCCESS;
12895 }
12896
12897 /*
12898 * Not a guest trap, must be a hypervisor related debug event then.
12899 * Update DR6 in case someone is interested in it.
12900 */
12901 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12902 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12903 CPUMSetHyperDR6(pVCpu, uDR6);
12904
12905 return rc;
12906}
12907
12908/**
12909 * VM-exit exception handler for \#GP (General-protection exception).
12910 *
12911 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12912 */
12913static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12914{
12915 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12917
12918 int rc;
12919 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12920 { /* likely */ }
12921 else
12922 {
12923#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12924 Assert(pVCpu->hm.s.fUsingDebugLoop);
12925#endif
12926 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12927 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12928 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12929 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12930 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12931 AssertRCReturn(rc, rc);
12932 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12933 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12934 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12935 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12936 return rc;
12937 }
12938
12939 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12940 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12941
12942 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12943 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12944 AssertRCReturn(rc, rc);
12945
12946 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12947 uint32_t cbOp = 0;
12948 PVM pVM = pVCpu->CTX_SUFF(pVM);
12949 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12950 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12951 if (RT_SUCCESS(rc))
12952 {
12953 rc = VINF_SUCCESS;
12954 Assert(cbOp == pDis->cbInstr);
12955 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12956 switch (pDis->pCurInstr->uOpcode)
12957 {
12958 case OP_CLI:
12959 {
12960 pMixedCtx->eflags.Bits.u1IF = 0;
12961 pMixedCtx->eflags.Bits.u1RF = 0;
12962 pMixedCtx->rip += pDis->cbInstr;
12963 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12964 if ( !fDbgStepping
12965 && pMixedCtx->eflags.Bits.u1TF)
12966 {
12967 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12968 AssertRCReturn(rc, rc);
12969 }
12970 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12971 break;
12972 }
12973
12974 case OP_STI:
12975 {
12976 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12977 pMixedCtx->eflags.Bits.u1IF = 1;
12978 pMixedCtx->eflags.Bits.u1RF = 0;
12979 pMixedCtx->rip += pDis->cbInstr;
12980 if (!fOldIF)
12981 {
12982 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12983 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12984 }
12985 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12986 if ( !fDbgStepping
12987 && pMixedCtx->eflags.Bits.u1TF)
12988 {
12989 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12990 AssertRCReturn(rc, rc);
12991 }
12992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12993 break;
12994 }
12995
12996 case OP_HLT:
12997 {
12998 rc = VINF_EM_HALT;
12999 pMixedCtx->rip += pDis->cbInstr;
13000 pMixedCtx->eflags.Bits.u1RF = 0;
13001 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13002 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13003 break;
13004 }
13005
13006 case OP_POPF:
13007 {
13008 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13009 uint32_t cbParm;
13010 uint32_t uMask;
13011 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13012 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13013 {
13014 cbParm = 4;
13015 uMask = 0xffffffff;
13016 }
13017 else
13018 {
13019 cbParm = 2;
13020 uMask = 0xffff;
13021 }
13022
13023 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13024 RTGCPTR GCPtrStack = 0;
13025 X86EFLAGS Eflags;
13026 Eflags.u32 = 0;
13027 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13028 &GCPtrStack);
13029 if (RT_SUCCESS(rc))
13030 {
13031 Assert(sizeof(Eflags.u32) >= cbParm);
13032 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13033 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13034 }
13035 if (RT_FAILURE(rc))
13036 {
13037 rc = VERR_EM_INTERPRETER;
13038 break;
13039 }
13040 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13041 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13042 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13043 pMixedCtx->esp += cbParm;
13044 pMixedCtx->esp &= uMask;
13045 pMixedCtx->rip += pDis->cbInstr;
13046 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13047 | HM_CHANGED_GUEST_RSP
13048 | HM_CHANGED_GUEST_RFLAGS);
13049 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13050 POPF restores EFLAGS.TF. */
13051 if ( !fDbgStepping
13052 && fGstStepping)
13053 {
13054 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13055 AssertRCReturn(rc, rc);
13056 }
13057 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13058 break;
13059 }
13060
13061 case OP_PUSHF:
13062 {
13063 uint32_t cbParm;
13064 uint32_t uMask;
13065 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13066 {
13067 cbParm = 4;
13068 uMask = 0xffffffff;
13069 }
13070 else
13071 {
13072 cbParm = 2;
13073 uMask = 0xffff;
13074 }
13075
13076 /* Get the stack pointer & push the contents of eflags onto the stack. */
13077 RTGCPTR GCPtrStack = 0;
13078 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13079 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13080 if (RT_FAILURE(rc))
13081 {
13082 rc = VERR_EM_INTERPRETER;
13083 break;
13084 }
13085 X86EFLAGS Eflags = pMixedCtx->eflags;
13086 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13087 Eflags.Bits.u1RF = 0;
13088 Eflags.Bits.u1VM = 0;
13089
13090 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13091 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13092 {
13093 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13094 rc = VERR_EM_INTERPRETER;
13095 break;
13096 }
13097 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13098 pMixedCtx->esp -= cbParm;
13099 pMixedCtx->esp &= uMask;
13100 pMixedCtx->rip += pDis->cbInstr;
13101 pMixedCtx->eflags.Bits.u1RF = 0;
13102 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13103 | HM_CHANGED_GUEST_RSP
13104 | HM_CHANGED_GUEST_RFLAGS);
13105 if ( !fDbgStepping
13106 && pMixedCtx->eflags.Bits.u1TF)
13107 {
13108 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13109 AssertRCReturn(rc, rc);
13110 }
13111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13112 break;
13113 }
13114
13115 case OP_IRET:
13116 {
13117 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13118 * instruction reference. */
13119 RTGCPTR GCPtrStack = 0;
13120 uint32_t uMask = 0xffff;
13121 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13122 uint16_t aIretFrame[3];
13123 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13124 {
13125 rc = VERR_EM_INTERPRETER;
13126 break;
13127 }
13128 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13129 &GCPtrStack);
13130 if (RT_SUCCESS(rc))
13131 {
13132 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13133 PGMACCESSORIGIN_HM));
13134 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13135 }
13136 if (RT_FAILURE(rc))
13137 {
13138 rc = VERR_EM_INTERPRETER;
13139 break;
13140 }
13141 pMixedCtx->eip = 0;
13142 pMixedCtx->ip = aIretFrame[0];
13143 pMixedCtx->cs.Sel = aIretFrame[1];
13144 pMixedCtx->cs.ValidSel = aIretFrame[1];
13145 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13146 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13147 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13148 pMixedCtx->sp += sizeof(aIretFrame);
13149 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13150 | HM_CHANGED_GUEST_CS
13151 | HM_CHANGED_GUEST_RSP
13152 | HM_CHANGED_GUEST_RFLAGS);
13153 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13154 if ( !fDbgStepping
13155 && fGstStepping)
13156 {
13157 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13158 AssertRCReturn(rc, rc);
13159 }
13160 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13162 break;
13163 }
13164
13165 case OP_INT:
13166 {
13167 uint16_t uVector = pDis->Param1.uValue & 0xff;
13168 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13169 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13171 break;
13172 }
13173
13174 case OP_INTO:
13175 {
13176 if (pMixedCtx->eflags.Bits.u1OF)
13177 {
13178 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13179 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13180 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13181 }
13182 else
13183 {
13184 pMixedCtx->eflags.Bits.u1RF = 0;
13185 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13186 }
13187 break;
13188 }
13189
13190 default:
13191 {
13192 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13193 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13194 EMCODETYPE_SUPERVISOR);
13195 rc = VBOXSTRICTRC_VAL(rc2);
13196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13197 /** @todo We have to set pending-debug exceptions here when the guest is
13198 * single-stepping depending on the instruction that was interpreted. */
13199 Log4Func(("#GP rc=%Rrc\n", rc));
13200 break;
13201 }
13202 }
13203 }
13204 else
13205 rc = VERR_EM_INTERPRETER;
13206
13207 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13208 ("#GP Unexpected rc=%Rrc\n", rc));
13209 return rc;
13210}
13211
13212
13213/**
13214 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13215 * the exception reported in the VMX transient structure back into the VM.
13216 *
13217 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13218 * up-to-date.
13219 */
13220static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13221{
13222 RT_NOREF_PV(pMixedCtx);
13223 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13224#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13225 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13226 ("uVector=%#x u32XcptBitmap=%#X32\n",
13227 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13228#endif
13229
13230 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13231 hmR0VmxCheckExitDueToEventDelivery(). */
13232 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13233 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13234 AssertRCReturn(rc, rc);
13235 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13236
13237#ifdef DEBUG_ramshankar
13238 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13239 | CPUMCTX_EXTRN_RIP);
13240 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13241 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13242#endif
13243
13244 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13245 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13246 return VINF_SUCCESS;
13247}
13248
13249
13250/**
13251 * VM-exit exception handler for \#PF (Page-fault exception).
13252 */
13253static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13254{
13255 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13256 PVM pVM = pVCpu->CTX_SUFF(pVM);
13257 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13258 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13259 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13260 AssertRCReturn(rc, rc);
13261
13262 if (!pVM->hm.s.fNestedPaging)
13263 { /* likely */ }
13264 else
13265 {
13266#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13267 Assert(pVCpu->hm.s.fUsingDebugLoop);
13268#endif
13269 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13270 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13271 {
13272 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13273 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13274 }
13275 else
13276 {
13277 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13278 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13279 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13280 }
13281 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13282 return rc;
13283 }
13284
13285 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13286 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13287 if (pVmxTransient->fVectoringPF)
13288 {
13289 Assert(pVCpu->hm.s.Event.fPending);
13290 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13291 }
13292
13293 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13294 AssertRCReturn(rc, rc);
13295
13296 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13297 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13298
13299 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13300 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13301 (RTGCPTR)pVmxTransient->uExitQualification);
13302
13303 Log4Func(("#PF: rc=%Rrc\n", rc));
13304 if (rc == VINF_SUCCESS)
13305 {
13306 /*
13307 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13308 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13309 */
13310 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13311 TRPMResetTrap(pVCpu);
13312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13313 return rc;
13314 }
13315
13316 if (rc == VINF_EM_RAW_GUEST_TRAP)
13317 {
13318 if (!pVmxTransient->fVectoringDoublePF)
13319 {
13320 /* It's a guest page fault and needs to be reflected to the guest. */
13321 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13322 TRPMResetTrap(pVCpu);
13323 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13324 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13325 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13326 }
13327 else
13328 {
13329 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13330 TRPMResetTrap(pVCpu);
13331 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13332 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13333 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13334 }
13335
13336 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13337 return VINF_SUCCESS;
13338 }
13339
13340 TRPMResetTrap(pVCpu);
13341 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13342 return rc;
13343}
13344
13345/** @} */
13346
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette