VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72990

Last change on this file since 72990 was 72990, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Don't update exit-history PC while exporting guest-state during VM-entry, we already update it on VM-exit
while importing state.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 563.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72990 2018-07-09 02:36:37Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Loads a set of guests MSRs to allow read/passthru to the guest.
1524 *
1525 * The name of this function is slightly confusing. This function does NOT
1526 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527 * common prefix for functions dealing with "lazy restoration" of the shared
1528 * MSRs.
1529 *
1530 * @param pVCpu The cross context virtual CPU structure.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 /*
1544 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546 * we can skip a few MSR writes.
1547 *
1548 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549 * guest MSR values in the guest-CPU context might be different to what's currently
1550 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551 * CPU, see @bugref{8728}.
1552 */
1553 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559 {
1560#ifdef VBOX_STRICT
1561 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565#endif
1566 }
1567 else
1568 {
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573 }
1574 }
1575#endif
1576 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxImportGuestState()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598#if HC_ARCH_BITS == 64
1599 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600 {
1601 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605 }
1606#endif
1607 }
1608 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1609}
1610
1611
1612/**
1613 * Verifies that our cached values of the VMCS fields are all consistent with
1614 * what's actually present in the VMCS.
1615 *
1616 * @returns VBox status code.
1617 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619 * VMCS content. HMCPU error-field is
1620 * updated, see VMX_VCI_XXX.
1621 * @param pVCpu The cross context virtual CPU structure.
1622 */
1623static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624{
1625 uint32_t u32Val;
1626 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627 AssertRCReturn(rc, rc);
1628 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655 {
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662 }
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671 uint64_t u64Val;
1672 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673 AssertRCReturn(rc, rc);
1674 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679 return VINF_SUCCESS;
1680}
1681
1682
1683#ifdef VBOX_STRICT
1684/**
1685 * Verifies that our cached host EFER value has not changed
1686 * since we cached it.
1687 *
1688 * @param pVCpu The cross context virtual CPU structure.
1689 */
1690static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691{
1692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695 {
1696 uint64_t u64Val;
1697 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698 AssertRC(rc);
1699
1700 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702 }
1703}
1704
1705
1706/**
1707 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708 * VMCS are correct.
1709 *
1710 * @param pVCpu The cross context virtual CPU structure.
1711 */
1712static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713{
1714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716 /* Verify MSR counts in the VMCS are what we think it should be. */
1717 uint32_t cMsrs;
1718 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730 {
1731 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733 pGuestMsr->u32Msr, cMsrs));
1734
1735 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739 /* Verify that the permissions are as expected in the MSR bitmap. */
1740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741 {
1742 VMXMSREXITREAD enmRead;
1743 VMXMSREXITWRITE enmWrite;
1744 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747 {
1748 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750 }
1751 else
1752 {
1753 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756 pGuestMsr->u32Msr, cMsrs));
1757 }
1758 }
1759 }
1760}
1761#endif /* VBOX_STRICT */
1762
1763
1764/**
1765 * Flushes the TLB using EPT.
1766 *
1767 * @returns VBox status code.
1768 * @param pVCpu The cross context virtual CPU structure of the calling
1769 * EMT. Can be NULL depending on @a enmTlbFlush.
1770 * @param enmTlbFlush Type of flush.
1771 *
1772 * @remarks Caller is responsible for making sure this function is called only
1773 * when NestedPaging is supported and providing @a enmTlbFlush that is
1774 * supported by the CPU.
1775 * @remarks Can be called with interrupts disabled.
1776 */
1777static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778{
1779 uint64_t au64Descriptor[2];
1780 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781 au64Descriptor[0] = 0;
1782 else
1783 {
1784 Assert(pVCpu);
1785 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786 }
1787 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790 AssertMsg(rc == VINF_SUCCESS,
1791 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793 if ( RT_SUCCESS(rc)
1794 && pVCpu)
1795 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796}
1797
1798
1799/**
1800 * Flushes the TLB using VPID.
1801 *
1802 * @returns VBox status code.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmTlbFlush.
1805 * @param enmTlbFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmTlbFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812{
1813 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834 if ( RT_SUCCESS(rc)
1835 && pVCpu)
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 NOREF(rc);
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVCpu The cross context virtual CPU structure.
1847 * @param GCVirt Guest virtual address of the page to invalidate.
1848 */
1849VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850{
1851 AssertPtr(pVCpu);
1852 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 if (!fFlushPending)
1856 {
1857 /*
1858 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859 * the EPT case. See @bugref{6043} and @bugref{6177}.
1860 *
1861 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862 * as this function maybe called in a loop with individual addresses.
1863 */
1864 PVM pVM = pVCpu->CTX_SUFF(pVM);
1865 if (pVM->hm.s.vmx.fVpid)
1866 {
1867 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870 /*
1871 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872 * where executing INVVPID outside 64-bit mode does not flush translations of
1873 * 64-bit linear addresses, see @bugref{6208#c72}.
1874 */
1875 if (RT_HI_U32(GCVirt))
1876 fVpidFlush = false;
1877#endif
1878
1879 if (fVpidFlush)
1880 {
1881 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883 }
1884 else
1885 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886 }
1887 else if (pVM->hm.s.fNestedPaging)
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897 * case where neither EPT nor VPID is supported by the CPU.
1898 *
1899 * @param pVCpu The cross context virtual CPU structure.
1900 * @param pCpu Pointer to the global HM struct.
1901 *
1902 * @remarks Called with interrupts disabled.
1903 */
1904static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905{
1906 AssertPtr(pVCpu);
1907 AssertPtr(pCpu);
1908
1909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914 pVCpu->hm.s.fForceTLBFlush = false;
1915 return;
1916}
1917
1918
1919/**
1920 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921 *
1922 * @param pVCpu The cross context virtual CPU structure.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926 * nomenclature. The reason is, to avoid confusion in compare statements
1927 * since the host-CPU copies are named "ASID".
1928 *
1929 * @remarks Called with interrupts disabled.
1930 */
1931static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932{
1933#ifdef VBOX_WITH_STATISTICS
1934 bool fTlbFlushed = false;
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937 if (!fTlbFlushed) \
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939 } while (0)
1940#else
1941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943#endif
1944
1945 AssertPtr(pCpu);
1946 AssertPtr(pVCpu);
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949 PVM pVM = pVCpu->CTX_SUFF(pVM);
1950 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954 /*
1955 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958 * cannot reuse the current ASID anymore.
1959 */
1960 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962 {
1963 ++pCpu->uCurrentAsid;
1964 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965 {
1966 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969 }
1970
1971 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975 /*
1976 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978 */
1979 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981 HMVMX_SET_TAGGED_TLB_FLUSHED();
1982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1983 }
1984
1985 /* Check for explicit TLB flushes. */
1986 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1987 {
1988 /*
1989 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1990 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1991 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1992 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1993 * mappings, see @bugref{6568}.
1994 *
1995 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1996 */
1997 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1998 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1999 HMVMX_SET_TAGGED_TLB_FLUSHED();
2000 }
2001
2002 pVCpu->hm.s.fForceTLBFlush = false;
2003 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2004
2005 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2006 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2007 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2008 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2009 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2010 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2011 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2012 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2013 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2014
2015 /* Update VMCS with the VPID. */
2016 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2017 AssertRC(rc);
2018
2019#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2020}
2021
2022
2023/**
2024 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2025 *
2026 * @returns VBox status code.
2027 * @param pVCpu The cross context virtual CPU structure.
2028 * @param pCpu Pointer to the global HM CPU struct.
2029 *
2030 * @remarks Called with interrupts disabled.
2031 */
2032static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2033{
2034 AssertPtr(pVCpu);
2035 AssertPtr(pCpu);
2036 Assert(pCpu->idCpu != NIL_RTCPUID);
2037 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2038 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2039
2040 /*
2041 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2042 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2043 */
2044 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2045 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2049 }
2050
2051 /* Check for explicit TLB flushes. */
2052 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2053 {
2054 pVCpu->hm.s.fForceTLBFlush = true;
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2056 }
2057
2058 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2059 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2060
2061 if (pVCpu->hm.s.fForceTLBFlush)
2062 {
2063 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2064 pVCpu->hm.s.fForceTLBFlush = false;
2065 }
2066}
2067
2068
2069/**
2070 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2071 *
2072 * @returns VBox status code.
2073 * @param pVCpu The cross context virtual CPU structure.
2074 * @param pCpu Pointer to the global HM CPU struct.
2075 *
2076 * @remarks Called with interrupts disabled.
2077 */
2078static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2079{
2080 AssertPtr(pVCpu);
2081 AssertPtr(pCpu);
2082 Assert(pCpu->idCpu != NIL_RTCPUID);
2083 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2084 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2085
2086 /*
2087 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2088 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2089 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2090 * cannot reuse the current ASID anymore.
2091 */
2092 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2093 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2094 {
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2097 }
2098
2099 /* Check for explicit TLB flushes. */
2100 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2101 {
2102 /*
2103 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2104 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2105 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2106 * include fExplicitFlush's too) - an obscure corner case.
2107 */
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2110 }
2111
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 ++pCpu->uCurrentAsid;
2117 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2118 {
2119 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2120 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2121 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2122 }
2123
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2126 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2127 if (pCpu->fFlushAsidBeforeUse)
2128 {
2129 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2130 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2131 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2132 {
2133 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2134 pCpu->fFlushAsidBeforeUse = false;
2135 }
2136 else
2137 {
2138 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2139 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2140 }
2141 }
2142 }
2143
2144 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2145 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2146 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2147 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2148 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2149 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2150 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2151
2152 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2153 AssertRC(rc);
2154}
2155
2156
2157/**
2158 * Flushes the guest TLB entry based on CPU capabilities.
2159 *
2160 * @param pVCpu The cross context virtual CPU structure.
2161 * @param pCpu Pointer to the global HM CPU struct.
2162 */
2163DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2164{
2165#ifdef HMVMX_ALWAYS_FLUSH_TLB
2166 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2167#endif
2168 PVM pVM = pVCpu->CTX_SUFF(pVM);
2169 switch (pVM->hm.s.vmx.enmTlbFlushType)
2170 {
2171 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2174 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2175 default:
2176 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2177 break;
2178 }
2179 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2180}
2181
2182
2183/**
2184 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2185 * TLB entries from the host TLB before VM-entry.
2186 *
2187 * @returns VBox status code.
2188 * @param pVM The cross context VM structure.
2189 */
2190static int hmR0VmxSetupTaggedTlb(PVM pVM)
2191{
2192 /*
2193 * Determine optimal flush type for Nested Paging.
2194 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2195 * guest execution (see hmR3InitFinalizeR0()).
2196 */
2197 if (pVM->hm.s.fNestedPaging)
2198 {
2199 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2200 {
2201 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2203 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2204 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2205 else
2206 {
2207 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2208 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2209 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2210 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211 }
2212
2213 /* Make sure the write-back cacheable memory type for EPT is supported. */
2214 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2215 {
2216 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2217 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219 }
2220
2221 /* EPT requires a page-walk length of 4. */
2222 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2223 {
2224 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2225 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2226 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227 }
2228 }
2229 else
2230 {
2231 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2232 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2233 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2234 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235 }
2236 }
2237
2238 /*
2239 * Determine optimal flush type for VPID.
2240 */
2241 if (pVM->hm.s.vmx.fVpid)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2253 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2255 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2256 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2257 pVM->hm.s.vmx.fVpid = false;
2258 }
2259 }
2260 else
2261 {
2262 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2263 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2264 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2265 pVM->hm.s.vmx.fVpid = false;
2266 }
2267 }
2268
2269 /*
2270 * Setup the handler for flushing tagged-TLBs.
2271 */
2272 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2274 else if (pVM->hm.s.fNestedPaging)
2275 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2276 else if (pVM->hm.s.vmx.fVpid)
2277 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2278 else
2279 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2280 return VINF_SUCCESS;
2281}
2282
2283
2284/**
2285 * Sets up pin-based VM-execution controls in the VMCS.
2286 *
2287 * @returns VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure.
2289 *
2290 * @remarks We don't really care about optimizing vmwrites here as it's done only
2291 * once per VM and hence we don't care about VMCS-field cache comparisons.
2292 */
2293static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2294{
2295 PVM pVM = pVCpu->CTX_SUFF(pVM);
2296 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2298
2299 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2300 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2301
2302 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2303 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2304
2305 /* Enable the VMX preemption timer. */
2306 if (pVM->hm.s.vmx.fUsePreemptTimer)
2307 {
2308 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2309 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2310 }
2311
2312#if 0
2313 /* Enable posted-interrupt processing. */
2314 if (pVM->hm.s.fPostedIntrs)
2315 {
2316 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2317 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2318 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2319 }
2320#endif
2321
2322 if ((fVal & fZap) != fVal)
2323 {
2324 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2325 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2328 }
2329
2330 /* Commit it to the VMCS and update our cache. */
2331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2332 AssertRCReturn(rc, rc);
2333 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2334
2335 return VINF_SUCCESS;
2336}
2337
2338
2339/**
2340 * Sets up secondary processor-based VM-execution controls in the VMCS.
2341 *
2342 * @returns VBox status code.
2343 * @param pVCpu The cross context virtual CPU structure.
2344 *
2345 * @remarks We don't really care about optimizing vmwrites here as it's done only
2346 * once per VM and hence we don't care about VMCS-field cache comparisons.
2347 */
2348static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2349{
2350 PVM pVM = pVCpu->CTX_SUFF(pVM);
2351 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354 /* WBINVD causes a VM-exit. */
2355 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2357
2358 /* Enable EPT (aka nested-paging). */
2359 if (pVM->hm.s.fNestedPaging)
2360 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2361
2362 /*
2363 * Enable the INVPCID instruction if supported by the hardware and we expose
2364 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2365 */
2366 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2367 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2369
2370 /* Enable VPID. */
2371 if (pVM->hm.s.vmx.fVpid)
2372 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2373
2374 /* Enable Unrestricted guest execution. */
2375 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2376 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2377
2378#if 0
2379 if (pVM->hm.s.fVirtApicRegs)
2380 {
2381 /* Enable APIC-register virtualization. */
2382 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2384
2385 /* Enable virtual-interrupt delivery. */
2386 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2387 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2388 }
2389#endif
2390
2391 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2392 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2393 * done dynamically. */
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2395 {
2396 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2397 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2399 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2400 AssertRCReturn(rc, rc);
2401 }
2402
2403 /* Enable RDTSCP. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2405 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2406
2407 /* Enable Pause-Loop exiting. */
2408 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2409 && pVM->hm.s.vmx.cPleGapTicks
2410 && pVM->hm.s.vmx.cPleWindowTicks)
2411 {
2412 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2413
2414 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2415 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2416 AssertRCReturn(rc, rc);
2417 }
2418
2419 if ((fVal & fZap) != fVal)
2420 {
2421 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2422 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2423 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2424 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2425 }
2426
2427 /* Commit it to the VMCS and update our cache. */
2428 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2429 AssertRCReturn(rc, rc);
2430 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2431
2432 return VINF_SUCCESS;
2433}
2434
2435
2436/**
2437 * Sets up processor-based VM-execution controls in the VMCS.
2438 *
2439 * @returns VBox status code.
2440 * @param pVCpu The cross context virtual CPU structure.
2441 *
2442 * @remarks We don't really care about optimizing vmwrites here as it's done only
2443 * once per VM and hence we don't care about VMCS-field cache comparisons.
2444 */
2445static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2446{
2447 PVM pVM = pVCpu->CTX_SUFF(pVM);
2448 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2449 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2452 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2453 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2454 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2455 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2456 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2458
2459 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2460 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2461 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2462 {
2463 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2464 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2465 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2466 }
2467
2468 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2469 if (!pVM->hm.s.fNestedPaging)
2470 {
2471 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2473 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2474 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2475 }
2476
2477 /* Use TPR shadowing if supported by the CPU. */
2478 if ( PDMHasApic(pVM)
2479 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480 {
2481 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2484 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2485 AssertRCReturn(rc, rc);
2486
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2488 /* CR8 writes cause a VM-exit based on TPR threshold. */
2489 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2490 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2491 }
2492 else
2493 {
2494 /*
2495 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2496 * Set this control only for 64-bit guests.
2497 */
2498 if (pVM->hm.s.fAllow64BitGuests)
2499 {
2500 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2501 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2502 }
2503 }
2504
2505 /* Use MSR-bitmaps if supported by the CPU. */
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2507 {
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2509
2510 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2512 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2513 AssertRCReturn(rc, rc);
2514
2515 /*
2516 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2517 * automatically using dedicated fields in the VMCS.
2518 */
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524#if HC_ARCH_BITS == 64
2525 /*
2526 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2527 */
2528 if (pVM->hm.s.fAllow64BitGuests)
2529 {
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534 }
2535#endif
2536 /*
2537 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2538 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2539 */
2540 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2541 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542
2543 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2544 }
2545
2546 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2547 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2549
2550 if ((fVal & fZap) != fVal)
2551 {
2552 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2553 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Commit it to the VMCS and update our cache. */
2559 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2560 AssertRCReturn(rc, rc);
2561 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2562
2563 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2564 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2565 return hmR0VmxSetupProcCtls2(pVCpu);
2566
2567 /* Sanity check, should not really happen. */
2568 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569 {
2570 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2571 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573 }
2574
2575 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2576 return VINF_SUCCESS;
2577}
2578
2579
2580/**
2581 * Sets up miscellaneous (everything other than Pin & Processor-based
2582 * VM-execution) control fields in the VMCS.
2583 *
2584 * @returns VBox status code.
2585 * @param pVCpu The cross context virtual CPU structure.
2586 */
2587static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2588{
2589 AssertPtr(pVCpu);
2590
2591 int rc = VERR_GENERAL_FAILURE;
2592
2593 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594#if 0
2595 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2596 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2598
2599 /*
2600 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2601 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2602 * We thus use the exception bitmap to control it rather than use both.
2603 */
2604 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2606
2607 /* All IO & IOIO instructions cause VM-exits. */
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2609 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2610
2611 /* Initialize the MSR-bitmap area. */
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2614 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 /* Setup MSR auto-load/store area. */
2619 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623 AssertRCReturn(rc, rc);
2624
2625 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2626 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2628 AssertRCReturn(rc, rc);
2629
2630 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2632 AssertRCReturn(rc, rc);
2633
2634 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2635#if 0
2636 /* Setup debug controls */
2637 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2638 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2639 AssertRCReturn(rc, rc);
2640#endif
2641
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2648 *
2649 * We shall setup those exception intercepts that don't change during the
2650 * lifetime of the VM here. The rest are done dynamically while loading the
2651 * guest state.
2652 *
2653 * @returns VBox status code.
2654 * @param pVCpu The cross context virtual CPU structure.
2655 */
2656static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2657{
2658 AssertPtr(pVCpu);
2659
2660 uint32_t uXcptBitmap;
2661
2662 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2663 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2664
2665 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2666 and writes, and because recursive #DBs can cause the CPU hang, we must always
2667 intercept #DB. */
2668 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2669
2670 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2671 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2672 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2673
2674 /* Commit it to the VMCS. */
2675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2676 AssertRCReturn(rc, rc);
2677
2678 /* Update our cache of the exception bitmap. */
2679 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2740 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2741 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2742 */
2743 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2744 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2745 || !pVM->hm.s.vmx.pRealModeTSS))
2746 {
2747 LogRelFunc(("Invalid real-on-v86 state.\n"));
2748 return VERR_INTERNAL_ERROR;
2749 }
2750
2751 /* Initialize these always, see hmR3InitFinalizeR0().*/
2752 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2753 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2754
2755 /* Setup the tagged-TLB flush handlers. */
2756 int rc = hmR0VmxSetupTaggedTlb(pVM);
2757 if (RT_FAILURE(rc))
2758 {
2759 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2760 return rc;
2761 }
2762
2763 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2764 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2765#if HC_ARCH_BITS == 64
2766 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2768 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2769 {
2770 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2771 }
2772#endif
2773
2774 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2775 RTCCUINTREG uHostCR4 = ASMGetCR4();
2776 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2777 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2778
2779 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2780 {
2781 PVMCPU pVCpu = &pVM->aCpus[i];
2782 AssertPtr(pVCpu);
2783 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2784
2785 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2786 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2787
2788 /* Set revision dword at the beginning of the VMCS structure. */
2789 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2790
2791 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2792 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 /* Load this VMCS as the current VMCS. */
2797 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2799 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801 rc = hmR0VmxSetupPinCtls(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804
2805 rc = hmR0VmxSetupProcCtls(pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2807 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808
2809 rc = hmR0VmxSetupMiscCtls(pVCpu);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 rc = hmR0VmxInitXcptBitmap(pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2816
2817#if HC_ARCH_BITS == 32
2818 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821#endif
2822
2823 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2824 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2826 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2827
2828 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2829
2830 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2831 }
2832
2833 return VINF_SUCCESS;
2834}
2835
2836
2837/**
2838 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2839 * the VMCS.
2840 *
2841 * @returns VBox status code.
2842 */
2843static int hmR0VmxExportHostControlRegs(void)
2844{
2845 RTCCUINTREG uReg = ASMGetCR0();
2846 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847 AssertRCReturn(rc, rc);
2848
2849 uReg = ASMGetCR3();
2850 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2851 AssertRCReturn(rc, rc);
2852
2853 uReg = ASMGetCR4();
2854 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2855 AssertRCReturn(rc, rc);
2856 return rc;
2857}
2858
2859
2860/**
2861 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2862 * the host-state area in the VMCS.
2863 *
2864 * @returns VBox status code.
2865 * @param pVCpu The cross context virtual CPU structure.
2866 */
2867static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2868{
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxExportHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890
2891 /*
2892 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893 * should -not- save the messed up state without restoring the original host-state,
2894 * see @bugref{7240}.
2895 *
2896 * This apparently can happen (most likely the FPU changes), deal with it rather than
2897 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2898 */
2899 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2900 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2901 {
2902 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2903 pVCpu->idCpu));
2904 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2905 }
2906 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2907#else
2908 RT_NOREF(pVCpu);
2909#endif
2910
2911 /*
2912 * Host DS, ES, FS and GS segment registers.
2913 */
2914#if HC_ARCH_BITS == 64
2915 RTSEL uSelDS = ASMGetDS();
2916 RTSEL uSelES = ASMGetES();
2917 RTSEL uSelFS = ASMGetFS();
2918 RTSEL uSelGS = ASMGetGS();
2919#else
2920 RTSEL uSelDS = 0;
2921 RTSEL uSelES = 0;
2922 RTSEL uSelFS = 0;
2923 RTSEL uSelGS = 0;
2924#endif
2925
2926 /*
2927 * Host CS and SS segment registers.
2928 */
2929 RTSEL uSelCS = ASMGetCS();
2930 RTSEL uSelSS = ASMGetSS();
2931
2932 /*
2933 * Host TR segment register.
2934 */
2935 RTSEL uSelTR = ASMGetTR();
2936
2937#if HC_ARCH_BITS == 64
2938 /*
2939 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2940 * gain VM-entry and restore them before we get preempted.
2941 *
2942 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2943 */
2944 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2945 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2946 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2947 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2948# undef VMXLOCAL_ADJUST_HOST_SEG
2949#endif
2950
2951 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2952 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2953 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2954 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2955 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2956 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2957 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2958 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2959 Assert(uSelCS);
2960 Assert(uSelTR);
2961
2962 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2963#if 0
2964 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2965 Assert(uSelSS != 0);
2966#endif
2967
2968 /* Write these host selector fields into the host-state area in the VMCS. */
2969 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2970 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2971#if HC_ARCH_BITS == 64
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2975 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2976#else
2977 NOREF(uSelDS);
2978 NOREF(uSelES);
2979 NOREF(uSelFS);
2980 NOREF(uSelGS);
2981#endif
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2983 AssertRCReturn(rc, rc);
2984
2985 /*
2986 * Host GDTR and IDTR.
2987 */
2988 RTGDTR Gdtr;
2989 RTIDTR Idtr;
2990 RT_ZERO(Gdtr);
2991 RT_ZERO(Idtr);
2992 ASMGetGDTR(&Gdtr);
2993 ASMGetIDTR(&Idtr);
2994 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2995 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2996 AssertRCReturn(rc, rc);
2997
2998#if HC_ARCH_BITS == 64
2999 /*
3000 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3001 * them to the maximum limit (0xffff) on every VM-exit.
3002 */
3003 if (Gdtr.cbGdt != 0xffff)
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3005
3006 /*
3007 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3008 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3009 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3010 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3011 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3012 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3013 * at 0xffff on hosts where we are sure it won't cause trouble.
3014 */
3015# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3016 if (Idtr.cbIdt < 0x0fff)
3017# else
3018 if (Idtr.cbIdt != 0xffff)
3019# endif
3020 {
3021 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3022 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024 }
3025#endif
3026
3027 /*
3028 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3029 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3030 * RPL should be too in most cases.
3031 */
3032 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3033 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3034
3035 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3036#if HC_ARCH_BITS == 64
3037 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3038
3039 /*
3040 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3041 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3042 * restoration if the host has something else. Task switching is not supported in 64-bit
3043 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3044 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3045 *
3046 * [1] See Intel spec. 3.5 "System Descriptor Types".
3047 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3048 */
3049 PVM pVM = pVCpu->CTX_SUFF(pVM);
3050 Assert(pDesc->System.u4Type == 11);
3051 if ( pDesc->System.u16LimitLow != 0x67
3052 || pDesc->System.u4LimitHigh)
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3055 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3056 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3057 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3058 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3059 }
3060
3061 /*
3062 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3063 */
3064 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3065 {
3066 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3067 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3068 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3069 {
3070 /* The GDT is read-only but the writable GDT is available. */
3071 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3072 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3073 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3074 AssertRCReturn(rc, rc);
3075 }
3076 }
3077#else
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return VINF_SUCCESS;
3100}
3101
3102
3103/**
3104 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3105 * host-state area of the VMCS.
3106 *
3107 * Theses MSRs will be automatically restored on the host after every successful
3108 * VM-exit.
3109 *
3110 * @returns VBox status code.
3111 * @param pVCpu The cross context virtual CPU structure.
3112 *
3113 * @remarks No-long-jump zone!!!
3114 */
3115static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3116{
3117 AssertPtr(pVCpu);
3118 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3119
3120 /*
3121 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3122 * rather than swapping them on every VM-entry.
3123 */
3124 hmR0VmxLazySaveHostMsrs(pVCpu);
3125
3126 /*
3127 * Host Sysenter MSRs.
3128 */
3129 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3130#if HC_ARCH_BITS == 32
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3133#else
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3136#endif
3137 AssertRCReturn(rc, rc);
3138
3139 /*
3140 * Host EFER MSR.
3141 *
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3143 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3144 */
3145 PVM pVM = pVCpu->CTX_SUFF(pVM);
3146 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3147 {
3148 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3149 AssertRCReturn(rc, rc);
3150 }
3151
3152 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3153
3154 return VINF_SUCCESS;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3163 * hmR0VMxExportGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 *
3168 * @remarks Requires EFER, CR4.
3169 * @remarks No-long-jump zone!!!
3170 */
3171static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3172{
3173#ifdef HMVMX_ALWAYS_SWAP_EFER
3174 return true;
3175#endif
3176
3177 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3178#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3179 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3180 if (CPUMIsGuestInLongModeEx(pCtx))
3181 return false;
3182#endif
3183
3184 PVM pVM = pVCpu->CTX_SUFF(pVM);
3185 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3186 uint64_t const u64GuestEfer = pCtx->msrEFER;
3187
3188 /*
3189 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3190 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3191 */
3192 if ( CPUMIsGuestInLongModeEx(pCtx)
3193 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3194 {
3195 return true;
3196 }
3197
3198 /*
3199 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3200 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3201 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3202 */
3203 if ( (pCtx->cr4 & X86_CR4_PAE)
3204 && (pCtx->cr0 & X86_CR0_PG)
3205 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3206 {
3207 /* Assert that host is PAE capable. */
3208 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3209 return true;
3210 }
3211
3212 return false;
3213}
3214
3215
3216/**
3217 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3218 *
3219 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3220 * see Intel spec. 24.8.1 "VM-entry controls".
3221 *
3222 * @returns VBox status code.
3223 * @param pVCpu The cross context virtual CPU structure.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3229{
3230 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3231 {
3232 PVM pVM = pVCpu->CTX_SUFF(pVM);
3233 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3234 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3235
3236 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3237 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3238
3239 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3240 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3241 {
3242 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3243 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3244 }
3245 else
3246 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3247
3248 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3249 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3250 && hmR0VmxShouldSwapEferMsr(pVCpu))
3251 {
3252 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3253 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3254 }
3255
3256 /*
3257 * The following should -not- be set (since we're not in SMM mode):
3258 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3259 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3260 */
3261
3262 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3263 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3264
3265 if ((fVal & fZap) != fVal)
3266 {
3267 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3268 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3269 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3271 }
3272
3273 /* Commit it to the VMCS and update our cache. */
3274 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3275 {
3276 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3277 AssertRCReturn(rc, rc);
3278 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3279 }
3280
3281 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3282 }
3283 return VINF_SUCCESS;
3284}
3285
3286
3287/**
3288 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3289 *
3290 * @returns VBox status code.
3291 * @param pVCpu The cross context virtual CPU structure.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3296{
3297 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3298 {
3299 PVM pVM = pVCpu->CTX_SUFF(pVM);
3300 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3301 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3302
3303 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3304 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3305
3306 /*
3307 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3308 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3309 * hmR0VmxExportHostMsrs().
3310 */
3311#if HC_ARCH_BITS == 64
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314#else
3315 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3316 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3317 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3318 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3319 {
3320 /* The switcher returns to long mode, EFER is managed by the switcher. */
3321 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3322 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3323 }
3324 else
3325 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3326#endif
3327
3328 /* If the newer VMCS fields for managing EFER exists, use it. */
3329 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3330 && hmR0VmxShouldSwapEferMsr(pVCpu))
3331 {
3332 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3333 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3334 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3335 }
3336
3337 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3338 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3339
3340 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3341 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3342 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3343
3344 /* Enable saving of the VMX preemption timer value on VM-exit. */
3345 if ( pVM->hm.s.vmx.fUsePreemptTimer
3346 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3347 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3348
3349 if ((fVal & fZap) != fVal)
3350 {
3351 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3352 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3353 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3354 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355 }
3356
3357 /* Commit it to the VMCS and update our cache. */
3358 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3359 {
3360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3361 AssertRCReturn(rc, rc);
3362 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3363 }
3364
3365 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3366 }
3367 return VINF_SUCCESS;
3368}
3369
3370
3371/**
3372 * Sets the TPR threshold in the VMCS.
3373 *
3374 * @returns VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure.
3376 * @param u32TprThreshold The TPR threshold (task-priority class only).
3377 */
3378DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3379{
3380 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3381 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3382 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3383}
3384
3385
3386/**
3387 * Exports the guest APIC TPR state into the VMCS.
3388 *
3389 * @returns VBox status code.
3390 * @param pVCpu The cross context virtual CPU structure.
3391 *
3392 * @remarks No-long-jump zone!!!
3393 */
3394static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3395{
3396 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3397 {
3398 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3399
3400 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3401 && APICIsEnabled(pVCpu))
3402 {
3403 /*
3404 * Setup TPR shadowing.
3405 */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3418 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3419 * priority of the pending interrupt so we can deliver the interrupt. If there
3420 * are no interrupts pending, set threshold to 0 to not cause any
3421 * TPR-below-threshold VM-exits.
3422 */
3423 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3424 uint32_t u32TprThreshold = 0;
3425 if (fPendingIntr)
3426 {
3427 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3428 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3429 const uint8_t u8TprPriority = u8Tpr >> 4;
3430 if (u8PendingPriority <= u8TprPriority)
3431 u32TprThreshold = u8PendingPriority;
3432 }
3433
3434 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3435 AssertRCReturn(rc, rc);
3436 }
3437 }
3438 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3439 }
3440 return VINF_SUCCESS;
3441}
3442
3443
3444/**
3445 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3446 *
3447 * @returns Guest's interruptibility-state.
3448 * @param pVCpu The cross context virtual CPU structure.
3449 *
3450 * @remarks No-long-jump zone!!!
3451 */
3452static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3453{
3454 /*
3455 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3456 */
3457 uint32_t fIntrState = 0;
3458 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459 {
3460 /* If inhibition is active, RIP & RFLAGS should've been accessed
3461 (i.e. read previously from the VMCS or from ring-3). */
3462 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3463#ifdef VBOX_STRICT
3464 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3465 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3466#endif
3467 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pCtx->eflags.Bits.u1IF)
3470 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler
3478 * without executing guest code in VT-x, the flag's condition to be cleared is
3479 * met and thus the cleared state is correct.
3480 */
3481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3482 }
3483 }
3484
3485 /*
3486 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3487 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3488 * setting this would block host-NMIs and IRET will not clear the blocking.
3489 *
3490 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3491 */
3492 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3493 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3494 {
3495 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3496 }
3497
3498 return fIntrState;
3499}
3500
3501
3502/**
3503 * Exports the guest's interruptibility-state into the guest-state area in the
3504 * VMCS.
3505 *
3506 * @returns VBox status code.
3507 * @param pVCpu The cross context virtual CPU structure.
3508 * @param fIntrState The interruptibility-state to set.
3509 */
3510static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3511{
3512 NOREF(pVCpu);
3513 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3514 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3515 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3516}
3517
3518
3519/**
3520 * Exports the exception intercepts required for guest execution in the VMCS.
3521 *
3522 * @returns VBox status code.
3523 * @param pVCpu The cross context virtual CPU structure.
3524 *
3525 * @remarks No-long-jump zone!!!
3526 */
3527static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3528{
3529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3530 {
3531 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3532
3533 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3534 if (pVCpu->hm.s.fGIMTrapXcptUD)
3535 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3536#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3537 else
3538 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3539#endif
3540
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3542 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3543
3544 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3545 {
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3547 AssertRCReturn(rc, rc);
3548 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3549 }
3550
3551 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3552 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3553 }
3554 return VINF_SUCCESS;
3555}
3556
3557
3558/**
3559 * Exports the guest's RIP into the guest-state area in the VMCS.
3560 *
3561 * @returns VBox status code.
3562 * @param pVCpu The cross context virtual CPU structure.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3570 {
3571 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3572
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3574 AssertRCReturn(rc, rc);
3575
3576 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3577 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Exports the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 *
3589 * @remarks No-long-jump zone!!!
3590 */
3591static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3592{
3593 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3594 {
3595 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3596
3597 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3601 }
3602 return VINF_SUCCESS;
3603}
3604
3605
3606/**
3607 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 *
3612 * @remarks No-long-jump zone!!!
3613 */
3614static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3615{
3616 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617 {
3618 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3619
3620 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621 Let us assert it as such and use 32-bit VMWRITE. */
3622 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3623 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3624 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3625 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3626
3627 /*
3628 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3629 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3630 * can run the real-mode guest code under Virtual 8086 mode.
3631 */
3632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633 {
3634 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3637 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639 }
3640
3641 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3642 AssertRCReturn(rc, rc);
3643
3644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3645 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3646 }
3647 return VINF_SUCCESS;
3648}
3649
3650
3651/**
3652 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3653 *
3654 * The guest FPU state is always pre-loaded hence we don't need to bother about
3655 * sharing FPU related CR0 bits between the guest and host.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu The cross context virtual CPU structure.
3659 *
3660 * @remarks No-long-jump zone!!!
3661 */
3662static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3663{
3664 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3665 {
3666 PVM pVM = pVCpu->CTX_SUFF(pVM);
3667 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3668 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3669
3670 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3671 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3672
3673 /*
3674 * Setup VT-x's view of the guest CR0.
3675 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3676 */
3677 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3678 if (pVM->hm.s.fNestedPaging)
3679 {
3680 if (CPUMIsGuestPagingEnabled(pVCpu))
3681 {
3682 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3683 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3684 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3685 }
3686 else
3687 {
3688 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3689 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692
3693 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3694 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3695 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696 }
3697 else
3698 {
3699 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3700 u32GuestCr0 |= X86_CR0_WP;
3701 }
3702
3703 /*
3704 * Guest FPU bits.
3705 *
3706 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3707 * using CR0.TS.
3708 *
3709 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3710 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3711 */
3712 u32GuestCr0 |= X86_CR0_NE;
3713
3714 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3715 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3716
3717 /*
3718 * Update exception intercepts.
3719 */
3720 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3721 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3722 {
3723 Assert(PDMVmmDevHeapIsEnabled(pVM));
3724 Assert(pVM->hm.s.vmx.pRealModeTSS);
3725 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3726 }
3727 else
3728 {
3729 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3730 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3731 if (fInterceptMF)
3732 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3733 }
3734
3735 /* Additional intercepts for debugging, define these yourself explicitly. */
3736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3737 uXcptBitmap |= 0
3738 | RT_BIT(X86_XCPT_BP)
3739 | RT_BIT(X86_XCPT_DE)
3740 | RT_BIT(X86_XCPT_NM)
3741 | RT_BIT(X86_XCPT_TS)
3742 | RT_BIT(X86_XCPT_UD)
3743 | RT_BIT(X86_XCPT_NP)
3744 | RT_BIT(X86_XCPT_SS)
3745 | RT_BIT(X86_XCPT_GP)
3746 | RT_BIT(X86_XCPT_PF)
3747 | RT_BIT(X86_XCPT_MF)
3748 ;
3749#elif defined(HMVMX_ALWAYS_TRAP_PF)
3750 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3751#endif
3752 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3798 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3799 AssertRCReturn(rc, rc);
3800
3801 /* Update our caches. */
3802 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3803 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3804 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3805
3806 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3807
3808 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3809 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3810 }
3811
3812 return VINF_SUCCESS;
3813}
3814
3815
3816/**
3817 * Exports the guest control registers (CR3, CR4) into the guest-state area
3818 * in the VMCS.
3819 *
3820 * @returns VBox strict status code.
3821 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3822 * without unrestricted guest access and the VMMDev is not presently
3823 * mapped (e.g. EFI32).
3824 *
3825 * @param pVCpu The cross context virtual CPU structure.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3845
3846 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3847 if (pVM->hm.s.fNestedPaging)
3848 {
3849 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3850
3851 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3852 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3854 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3855
3856 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3857 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3858 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3859
3860 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3861 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3862 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3863 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3864 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3865 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3866 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867
3868 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3869 AssertRCReturn(rc, rc);
3870
3871 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3872 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3873 || CPUMIsGuestPagingEnabledEx(pCtx))
3874 {
3875 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3876 if (CPUMIsGuestInPAEModeEx(pCtx))
3877 {
3878 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3879 AssertRCReturn(rc, rc);
3880 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3883 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3884 AssertRCReturn(rc, rc);
3885 }
3886
3887 /*
3888 * The guest's view of its CR3 is unblemished with Nested Paging when the
3889 * guest is using paging or we have unrestricted guest execution to handle
3890 * the guest when it's not using paging.
3891 */
3892 GCPhysGuestCR3 = pCtx->cr3;
3893 }
3894 else
3895 {
3896 /*
3897 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3898 * thinks it accesses physical memory directly, we use our identity-mapped
3899 * page table to map guest-linear to guest-physical addresses. EPT takes care
3900 * of translating it to host-physical addresses.
3901 */
3902 RTGCPHYS GCPhys;
3903 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3904
3905 /* We obtain it here every time as the guest could have relocated this PCI region. */
3906 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907 if (RT_SUCCESS(rc))
3908 { /* likely */ }
3909 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3910 {
3911 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3912 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3913 }
3914 else
3915 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3916
3917 GCPhysGuestCR3 = GCPhys;
3918 }
3919
3920 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3921 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922 AssertRCReturn(rc, rc);
3923 }
3924 else
3925 {
3926 /* Non-nested paging case, just use the hypervisor's CR3. */
3927 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3928
3929 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3930 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3931 AssertRCReturn(rc, rc);
3932 }
3933
3934 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3935 }
3936
3937 /*
3938 * Guest CR4.
3939 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3940 */
3941 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3942 {
3943 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3944 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3945 Assert(!RT_HI_U32(pCtx->cr4));
3946
3947 uint32_t u32GuestCr4 = pCtx->cr4;
3948 uint32_t const u32ShadowCr4 = pCtx->cr4;
3949
3950 /*
3951 * Setup VT-x's view of the guest CR4.
3952 *
3953 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3954 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3955 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3956 *
3957 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958 */
3959 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960 {
3961 Assert(pVM->hm.s.vmx.pRealModeTSS);
3962 Assert(PDMVmmDevHeapIsEnabled(pVM));
3963 u32GuestCr4 &= ~X86_CR4_VME;
3964 }
3965
3966 if (pVM->hm.s.fNestedPaging)
3967 {
3968 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3969 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3970 {
3971 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972 u32GuestCr4 |= X86_CR4_PSE;
3973 /* Our identity mapping is a 32-bit page directory. */
3974 u32GuestCr4 &= ~X86_CR4_PAE;
3975 }
3976 /* else use guest CR4.*/
3977 }
3978 else
3979 {
3980 /*
3981 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983 */
3984 switch (pVCpu->hm.s.enmShadowMode)
3985 {
3986 case PGMMODE_REAL: /* Real-mode. */
3987 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988 case PGMMODE_32_BIT: /* 32-bit paging. */
3989 {
3990 u32GuestCr4 &= ~X86_CR4_PAE;
3991 break;
3992 }
3993
3994 case PGMMODE_PAE: /* PAE paging. */
3995 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996 {
3997 u32GuestCr4 |= X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004 break;
4005#endif
4006 default:
4007 AssertFailed();
4008 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009 }
4010 }
4011
4012 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 u32GuestCr4 |= fSetCr4;
4016 u32GuestCr4 &= fZapCr4;
4017
4018 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4019 that would cause a VM-exit. */
4020 uint32_t u32Cr4Mask = X86_CR4_VME
4021 | X86_CR4_PAE
4022 | X86_CR4_PGE
4023 | X86_CR4_PSE
4024 | X86_CR4_VMXE;
4025 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4026 u32Cr4Mask |= X86_CR4_OSXSAVE;
4027 if (pVM->cpum.ro.GuestFeatures.fPcid)
4028 u32Cr4Mask |= X86_CR4_PCIDE;
4029
4030 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4031 into the VMCS and update our cache. */
4032 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4034 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4035 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4036 AssertRCReturn(rc, rc);
4037 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4038
4039 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4040 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4041
4042 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4043
4044 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4045 fZapCr4));
4046 }
4047 return rc;
4048}
4049
4050
4051/**
4052 * Exports the guest debug registers into the guest-state area in the VMCS.
4053 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4054 *
4055 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4056 *
4057 * @returns VBox status code.
4058 * @param pVCpu The cross context virtual CPU structure.
4059 *
4060 * @remarks No-long-jump zone!!!
4061 */
4062static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4063{
4064 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4068 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4069 {
4070 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4072 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4073 }
4074#endif
4075
4076 bool fSteppingDB = false;
4077 bool fInterceptMovDRx = false;
4078 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4079 if (pVCpu->hm.s.fSingleInstruction)
4080 {
4081 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4082 PVM pVM = pVCpu->CTX_SUFF(pVM);
4083 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4084 {
4085 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4086 Assert(fSteppingDB == false);
4087 }
4088 else
4089 {
4090 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4091 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4092 pVCpu->hm.s.fClearTrapFlag = true;
4093 fSteppingDB = true;
4094 }
4095 }
4096
4097 uint32_t u32GuestDr7;
4098 if ( fSteppingDB
4099 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4100 {
4101 /*
4102 * Use the combined guest and host DRx values found in the hypervisor register set
4103 * because the debugger has breakpoints active or someone is single stepping on the
4104 * host side without a monitor trap flag.
4105 *
4106 * Note! DBGF expects a clean DR6 state before executing guest code.
4107 */
4108#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4110 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4111 {
4112 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4113 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4114 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4115 }
4116 else
4117#endif
4118 if (!CPUMIsHyperDebugStateActive(pVCpu))
4119 {
4120 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4121 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4122 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4123 }
4124
4125 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4126 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4127 pVCpu->hm.s.fUsingHyperDR7 = true;
4128 fInterceptMovDRx = true;
4129 }
4130 else
4131 {
4132 /*
4133 * If the guest has enabled debug registers, we need to load them prior to
4134 * executing guest code so they'll trigger at the right time.
4135 */
4136 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4137 {
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4140 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146 }
4147 else
4148#endif
4149 if (!CPUMIsGuestDebugStateActive(pVCpu))
4150 {
4151 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155 }
4156 Assert(!fInterceptMovDRx);
4157 }
4158 /*
4159 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160 * must intercept #DB in order to maintain a correct DR6 guest value, and
4161 * because we need to intercept it to prevent nested #DBs from hanging the
4162 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4163 */
4164#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4165 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4166 && !CPUMIsGuestDebugStateActive(pVCpu))
4167#else
4168 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4169#endif
4170 {
4171 fInterceptMovDRx = true;
4172 }
4173
4174 /* Update DR7 with the actual guest value. */
4175 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4176 pVCpu->hm.s.fUsingHyperDR7 = false;
4177 }
4178
4179 if (fInterceptMovDRx)
4180 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181 else
4182 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4183
4184 /*
4185 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4186 * monitor-trap flag and update our cache.
4187 */
4188 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4189 {
4190 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4191 AssertRCReturn(rc2, rc2);
4192 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4193 }
4194
4195 /*
4196 * Update guest DR7.
4197 */
4198 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4199 AssertRCReturn(rc, rc);
4200
4201 return VINF_SUCCESS;
4202}
4203
4204
4205#ifdef VBOX_STRICT
4206/**
4207 * Strict function to validate segment registers.
4208 *
4209 * @param pVCpu The cross context virtual CPU structure.
4210 *
4211 * @remarks Will import guest CR0 on strict builds during validation of
4212 * segments.
4213 */
4214static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4215{
4216 /*
4217 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4218 *
4219 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4220 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4221 * and doesn't change the guest-context value.
4222 */
4223 PVM pVM = pVCpu->CTX_SUFF(pVM);
4224 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4225 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4226 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4227 && ( !CPUMIsGuestInRealModeEx(pCtx)
4228 && !CPUMIsGuestInV86ModeEx(pCtx)))
4229 {
4230 /* Protected mode checks */
4231 /* CS */
4232 Assert(pCtx->cs.Attr.n.u1Present);
4233 Assert(!(pCtx->cs.Attr.u & 0xf00));
4234 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4235 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4236 || !(pCtx->cs.Attr.n.u1Granularity));
4237 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4238 || (pCtx->cs.Attr.n.u1Granularity));
4239 /* CS cannot be loaded with NULL in protected mode. */
4240 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4241 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4242 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4243 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4244 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4245 else
4246 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4247 /* SS */
4248 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4249 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4250 if ( !(pCtx->cr0 & X86_CR0_PE)
4251 || pCtx->cs.Attr.n.u4Type == 3)
4252 {
4253 Assert(!pCtx->ss.Attr.n.u2Dpl);
4254 }
4255 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4256 {
4257 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4258 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4259 Assert(pCtx->ss.Attr.n.u1Present);
4260 Assert(!(pCtx->ss.Attr.u & 0xf00));
4261 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->ss.Attr.n.u1Granularity));
4264 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4265 || (pCtx->ss.Attr.n.u1Granularity));
4266 }
4267 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4268 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4269 {
4270 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4271 Assert(pCtx->ds.Attr.n.u1Present);
4272 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4273 Assert(!(pCtx->ds.Attr.u & 0xf00));
4274 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4275 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4276 || !(pCtx->ds.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4278 || (pCtx->ds.Attr.n.u1Granularity));
4279 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4280 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4281 }
4282 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4285 Assert(pCtx->es.Attr.n.u1Present);
4286 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4287 Assert(!(pCtx->es.Attr.u & 0xf00));
4288 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->es.Attr.n.u1Granularity));
4291 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4292 || (pCtx->es.Attr.n.u1Granularity));
4293 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4294 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4295 }
4296 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4297 {
4298 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4299 Assert(pCtx->fs.Attr.n.u1Present);
4300 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4301 Assert(!(pCtx->fs.Attr.u & 0xf00));
4302 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4303 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4304 || !(pCtx->fs.Attr.n.u1Granularity));
4305 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4306 || (pCtx->fs.Attr.n.u1Granularity));
4307 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4308 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4309 }
4310 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4311 {
4312 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313 Assert(pCtx->gs.Attr.n.u1Present);
4314 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4315 Assert(!(pCtx->gs.Attr.u & 0xf00));
4316 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4317 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4318 || !(pCtx->gs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4320 || (pCtx->gs.Attr.n.u1Granularity));
4321 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323 }
4324 /* 64-bit capable CPUs. */
4325# if HC_ARCH_BITS == 64
4326 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4327 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4328 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4329 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4330# endif
4331 }
4332 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4333 || ( CPUMIsGuestInRealModeEx(pCtx)
4334 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4335 {
4336 /* Real and v86 mode checks. */
4337 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4338 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4339 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4340 {
4341 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4342 }
4343 else
4344 {
4345 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4346 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4347 }
4348
4349 /* CS */
4350 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4351 Assert(pCtx->cs.u32Limit == 0xffff);
4352 Assert(u32CSAttr == 0xf3);
4353 /* SS */
4354 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4355 Assert(pCtx->ss.u32Limit == 0xffff);
4356 Assert(u32SSAttr == 0xf3);
4357 /* DS */
4358 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4359 Assert(pCtx->ds.u32Limit == 0xffff);
4360 Assert(u32DSAttr == 0xf3);
4361 /* ES */
4362 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4363 Assert(pCtx->es.u32Limit == 0xffff);
4364 Assert(u32ESAttr == 0xf3);
4365 /* FS */
4366 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4367 Assert(pCtx->fs.u32Limit == 0xffff);
4368 Assert(u32FSAttr == 0xf3);
4369 /* GS */
4370 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4371 Assert(pCtx->gs.u32Limit == 0xffff);
4372 Assert(u32GSAttr == 0xf3);
4373 /* 64-bit capable CPUs. */
4374# if HC_ARCH_BITS == 64
4375 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4376 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4377 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4378 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4379# endif
4380 }
4381}
4382#endif /* VBOX_STRICT */
4383
4384
4385/**
4386 * Exports a guest segment register into the guest-state area in the VMCS.
4387 *
4388 * @returns VBox status code.
4389 * @param pVCpu The cross context virtual CPU structure.
4390 * @param idxSel Index of the selector in the VMCS.
4391 * @param idxLimit Index of the segment limit in the VMCS.
4392 * @param idxBase Index of the segment base in the VMCS.
4393 * @param idxAccess Index of the access rights of the segment in the VMCS.
4394 * @param pSelReg Pointer to the segment selector.
4395 *
4396 * @remarks No-long-jump zone!!!
4397 */
4398static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4399 PCCPUMSELREG pSelReg)
4400{
4401 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4402 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4403 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4404 AssertRCReturn(rc, rc);
4405
4406 uint32_t u32Access = pSelReg->Attr.u;
4407 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4408 {
4409 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4410 u32Access = 0xf3;
4411 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4412 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4413 }
4414 else
4415 {
4416 /*
4417 * The way to differentiate between whether this is really a null selector or was just
4418 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4419 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4420 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4421 * NULL selectors loaded in protected-mode have their attribute as 0.
4422 */
4423 if (!u32Access)
4424 u32Access = X86DESCATTR_UNUSABLE;
4425 }
4426
4427 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4428 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4429 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4430
4431 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4432 AssertRCReturn(rc, rc);
4433 return rc;
4434}
4435
4436
4437/**
4438 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4439 * into the guest-state area in the VMCS.
4440 *
4441 * @returns VBox status code.
4442 * @param pVCpu The cross context virtual CPU structure.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4453
4454 /*
4455 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4456 */
4457 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4458 {
4459#ifdef VBOX_WITH_REM
4460 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4461 {
4462 Assert(pVM->hm.s.vmx.pRealModeTSS);
4463 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4464 if ( pVCpu->hm.s.vmx.fWasInRealMode
4465 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4466 {
4467 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4468 in real-mode (e.g. OpenBSD 4.0) */
4469 REMFlushTBs(pVM);
4470 Log4Func(("Switch to protected mode detected!\n"));
4471 pVCpu->hm.s.vmx.fWasInRealMode = false;
4472 }
4473 }
4474#endif
4475 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4476 {
4477 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4478 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4480 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4481 AssertRCReturn(rc, rc);
4482 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4483 }
4484
4485 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4486 {
4487 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4488 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4490 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4491 AssertRCReturn(rc, rc);
4492 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4493 }
4494
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4496 {
4497 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4500 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4501 AssertRCReturn(rc, rc);
4502 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4503 }
4504
4505 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4506 {
4507 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4508 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4509 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4510 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4511 AssertRCReturn(rc, rc);
4512 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4513 }
4514
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4516 {
4517 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4519 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4520 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4521 AssertRCReturn(rc, rc);
4522 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4523 }
4524
4525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4526 {
4527 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4528 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4530 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4531 AssertRCReturn(rc, rc);
4532 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4533 }
4534
4535#ifdef VBOX_STRICT
4536 hmR0VmxValidateSegmentRegs(pVCpu);
4537#endif
4538
4539 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4540 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4541 }
4542
4543 /*
4544 * Guest TR.
4545 */
4546 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4547 {
4548 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4549
4550 /*
4551 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4552 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4553 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4554 */
4555 uint16_t u16Sel = 0;
4556 uint32_t u32Limit = 0;
4557 uint64_t u64Base = 0;
4558 uint32_t u32AccessRights = 0;
4559
4560 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4561 {
4562 u16Sel = pCtx->tr.Sel;
4563 u32Limit = pCtx->tr.u32Limit;
4564 u64Base = pCtx->tr.u64Base;
4565 u32AccessRights = pCtx->tr.Attr.u;
4566 }
4567 else
4568 {
4569 Assert(pVM->hm.s.vmx.pRealModeTSS);
4570 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4571
4572 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4573 RTGCPHYS GCPhys;
4574 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4575 AssertRCReturn(rc, rc);
4576
4577 X86DESCATTR DescAttr;
4578 DescAttr.u = 0;
4579 DescAttr.n.u1Present = 1;
4580 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4581
4582 u16Sel = 0;
4583 u32Limit = HM_VTX_TSS_SIZE;
4584 u64Base = GCPhys; /* in real-mode phys = virt. */
4585 u32AccessRights = DescAttr.u;
4586 }
4587
4588 /* Validate. */
4589 Assert(!(u16Sel & RT_BIT(2)));
4590 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4591 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4592 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4593 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4594 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4595 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4596 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4597 Assert( (u32Limit & 0xfff) == 0xfff
4598 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4599 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4600 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4601
4602 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4603 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4604 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4605 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4606 AssertRCReturn(rc, rc);
4607
4608 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4609 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4610 }
4611
4612 /*
4613 * Guest GDTR.
4614 */
4615 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4616 {
4617 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4618
4619 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4620 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4621 AssertRCReturn(rc, rc);
4622
4623 /* Validate. */
4624 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4627 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4628 }
4629
4630 /*
4631 * Guest LDTR.
4632 */
4633 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4634 {
4635 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4636
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4645 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4648 AssertRCReturn(rc, rc);
4649
4650 /* Validate. */
4651 if (!(u32Access & X86DESCATTR_UNUSABLE))
4652 {
4653 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4654 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4655 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4656 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4657 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4658 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4659 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4660 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4661 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4662 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4663 }
4664
4665 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4666 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4667 }
4668
4669 /*
4670 * Guest IDTR.
4671 */
4672 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4673 {
4674 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4675
4676 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4677 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4678 AssertRCReturn(rc, rc);
4679
4680 /* Validate. */
4681 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4682
4683 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4684 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4685 }
4686
4687 return VINF_SUCCESS;
4688}
4689
4690
4691/**
4692 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4693 * areas.
4694 *
4695 * These MSRs will automatically be loaded to the host CPU on every successful
4696 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4697 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4698 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4699 *
4700 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4701 *
4702 * @returns VBox status code.
4703 * @param pVCpu The cross context virtual CPU structure.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4715 */
4716 PVM pVM = pVCpu->CTX_SUFF(pVM);
4717 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4718 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4719 {
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722#if HC_ARCH_BITS == 32
4723 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4724
4725 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4727 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4728 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4729 AssertRCReturn(rc, rc);
4730# ifdef LOG_ENABLED
4731 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4732 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4733 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4734# endif
4735#endif
4736 }
4737 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4746 {
4747 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4748
4749 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4750 {
4751 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4752 AssertRCReturn(rc, rc);
4753 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4754 }
4755
4756 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4757 {
4758 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4759 AssertRCReturn(rc, rc);
4760 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4761 }
4762
4763 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4764 {
4765 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4766 AssertRCReturn(rc, rc);
4767 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4768 }
4769 }
4770
4771 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4772 {
4773 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4774
4775 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4776 {
4777 /*
4778 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4779 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4780 */
4781 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4782 {
4783 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4784 AssertRCReturn(rc,rc);
4785 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4786 }
4787 else
4788 {
4789 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4790 NULL /* pfAddedAndUpdated */);
4791 AssertRCReturn(rc, rc);
4792
4793 /* We need to intercept reads too, see @bugref{7386#c16}. */
4794 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4795 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4796 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4797 pVCpu->hm.s.vmx.cMsrs));
4798 }
4799 }
4800 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4801 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4802 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4803 }
4804
4805 return VINF_SUCCESS;
4806}
4807
4808
4809#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4810/**
4811 * Check if guest state allows safe use of 32-bit switcher again.
4812 *
4813 * Segment bases and protected mode structures must be 32-bit addressable
4814 * because the 32-bit switcher will ignore high dword when writing these VMCS
4815 * fields. See @bugref{8432} for details.
4816 *
4817 * @returns true if safe, false if must continue to use the 64-bit switcher.
4818 * @param pCtx Pointer to the guest-CPU context.
4819 *
4820 * @remarks No-long-jump zone!!!
4821 */
4822static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4823{
4824 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834
4835 /* All good, bases are 32-bit. */
4836 return true;
4837}
4838#endif
4839
4840
4841/**
4842 * Selects up the appropriate function to run guest code.
4843 *
4844 * @returns VBox status code.
4845 * @param pVCpu The cross context virtual CPU structure.
4846 *
4847 * @remarks No-long-jump zone!!!
4848 */
4849static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4850{
4851 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4852 if (CPUMIsGuestInLongModeEx(pCtx))
4853 {
4854#ifndef VBOX_ENABLE_64_BITS_GUESTS
4855 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4856#endif
4857 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4858#if HC_ARCH_BITS == 32
4859 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4860 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4861 {
4862#ifdef VBOX_STRICT
4863 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4864 {
4865 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4866 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4867 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4868 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4871 }
4872#endif
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4874
4875 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4876 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4877 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4878 Log4Func(("Selected 64-bit switcher\n"));
4879 }
4880#else
4881 /* 64-bit host. */
4882 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4883#endif
4884 }
4885 else
4886 {
4887 /* Guest is not in long mode, use the 32-bit handler. */
4888#if HC_ARCH_BITS == 32
4889 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4890 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4891 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4892 {
4893# ifdef VBOX_STRICT
4894 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4895 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4896 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4897 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4898 | HM_CHANGED_VMX_ENTRY_CTLS
4899 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4900# endif
4901 }
4902# ifdef VBOX_ENABLE_64_BITS_GUESTS
4903 /*
4904 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4905 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4906 * switcher flag because now we know the guest is in a sane state where it's safe
4907 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4908 * the much faster 32-bit switcher again.
4909 */
4910 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4911 {
4912 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4913 Log4Func(("Selected 32-bit switcher\n"));
4914 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4915 }
4916 else
4917 {
4918 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4919 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4920 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4921 {
4922 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4923 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4924 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4925 | HM_CHANGED_VMX_ENTRY_CTLS
4926 | HM_CHANGED_VMX_EXIT_CTLS
4927 | HM_CHANGED_HOST_CONTEXT);
4928 Log4Func(("Selected 32-bit switcher (safe)\n"));
4929 }
4930 }
4931# else
4932 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4933# endif
4934#else
4935 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4936#endif
4937 }
4938 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4939 return VINF_SUCCESS;
4940}
4941
4942
4943/**
4944 * Wrapper for running the guest code in VT-x.
4945 *
4946 * @returns VBox status code, no informational status codes.
4947 * @param pVCpu The cross context virtual CPU structure.
4948 *
4949 * @remarks No-long-jump zone!!!
4950 */
4951DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4952{
4953 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4954 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4955 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4956
4957 /*
4958 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4959 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4960 * callee-saved and thus the need for this XMM wrapper.
4961 *
4962 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4963 */
4964 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4965 /** @todo Add stats for resume vs launch. */
4966 PVM pVM = pVCpu->CTX_SUFF(pVM);
4967#ifdef VBOX_WITH_KERNEL_USING_XMM
4968 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4969#else
4970 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4971#endif
4972 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4973 return rc;
4974}
4975
4976
4977/**
4978 * Reports world-switch error and dumps some useful debug info.
4979 *
4980 * @param pVCpu The cross context virtual CPU structure.
4981 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4982 * @param pVmxTransient Pointer to the VMX transient structure (only
4983 * exitReason updated).
4984 */
4985static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4986{
4987 Assert(pVCpu);
4988 Assert(pVmxTransient);
4989 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4990
4991 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4992 switch (rcVMRun)
4993 {
4994 case VERR_VMX_INVALID_VMXON_PTR:
4995 AssertFailed();
4996 break;
4997 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4998 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4999 {
5000 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5001 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5002 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5003 AssertRC(rc);
5004
5005 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5006 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5007 Cannot do it here as we may have been long preempted. */
5008
5009#ifdef VBOX_STRICT
5010 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5011 pVmxTransient->uExitReason));
5012 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5013 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5014 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5015 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5016 else
5017 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5018 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5019 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5020
5021 /* VMX control bits. */
5022 uint32_t u32Val;
5023 uint64_t u64Val;
5024 RTHCUINTREG uHCReg;
5025 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5026 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5027 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5028 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5029 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5030 {
5031 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5032 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5033 }
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5061 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5062 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5063 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5065 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5066 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5067 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5068 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5069 {
5070 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5071 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5072 }
5073
5074 /* Guest bits. */
5075 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5076 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5077 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5078 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5079 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5080 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5081 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5082 {
5083 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5085 }
5086
5087 /* Host bits. */
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5089 Log4(("Host CR0 %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5091 Log4(("Host CR3 %#RHr\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5093 Log4(("Host CR4 %#RHr\n", uHCReg));
5094
5095 RTGDTR HostGdtr;
5096 PCX86DESCHC pDesc;
5097 ASMGetGDTR(&HostGdtr);
5098 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5099 Log4(("Host CS %#08x\n", u32Val));
5100 if (u32Val < HostGdtr.cbGdt)
5101 {
5102 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5103 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5104 }
5105
5106 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5107 Log4(("Host DS %#08x\n", u32Val));
5108 if (u32Val < HostGdtr.cbGdt)
5109 {
5110 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5111 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5112 }
5113
5114 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5115 Log4(("Host ES %#08x\n", u32Val));
5116 if (u32Val < HostGdtr.cbGdt)
5117 {
5118 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5119 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5120 }
5121
5122 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5123 Log4(("Host FS %#08x\n", u32Val));
5124 if (u32Val < HostGdtr.cbGdt)
5125 {
5126 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5127 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5128 }
5129
5130 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5131 Log4(("Host GS %#08x\n", u32Val));
5132 if (u32Val < HostGdtr.cbGdt)
5133 {
5134 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5135 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5136 }
5137
5138 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5139 Log4(("Host SS %#08x\n", u32Val));
5140 if (u32Val < HostGdtr.cbGdt)
5141 {
5142 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5143 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5144 }
5145
5146 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5147 Log4(("Host TR %#08x\n", u32Val));
5148 if (u32Val < HostGdtr.cbGdt)
5149 {
5150 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5151 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5152 }
5153
5154 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5155 Log4(("Host TR Base %#RHv\n", uHCReg));
5156 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5157 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5158 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5159 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5160 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5161 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5163 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5165 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5167 Log4(("Host RSP %#RHv\n", uHCReg));
5168 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5169 Log4(("Host RIP %#RHv\n", uHCReg));
5170# if HC_ARCH_BITS == 64
5171 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5172 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5173 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5174 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5175 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5176 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5177# endif
5178#endif /* VBOX_STRICT */
5179 break;
5180 }
5181
5182 default:
5183 /* Impossible */
5184 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5185 break;
5186 }
5187}
5188
5189
5190#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5191#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5192# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5193#endif
5194#ifdef VBOX_STRICT
5195static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5196{
5197 switch (idxField)
5198 {
5199 case VMX_VMCS_GUEST_RIP:
5200 case VMX_VMCS_GUEST_RSP:
5201 case VMX_VMCS_GUEST_SYSENTER_EIP:
5202 case VMX_VMCS_GUEST_SYSENTER_ESP:
5203 case VMX_VMCS_GUEST_GDTR_BASE:
5204 case VMX_VMCS_GUEST_IDTR_BASE:
5205 case VMX_VMCS_GUEST_CS_BASE:
5206 case VMX_VMCS_GUEST_DS_BASE:
5207 case VMX_VMCS_GUEST_ES_BASE:
5208 case VMX_VMCS_GUEST_FS_BASE:
5209 case VMX_VMCS_GUEST_GS_BASE:
5210 case VMX_VMCS_GUEST_SS_BASE:
5211 case VMX_VMCS_GUEST_LDTR_BASE:
5212 case VMX_VMCS_GUEST_TR_BASE:
5213 case VMX_VMCS_GUEST_CR3:
5214 return true;
5215 }
5216 return false;
5217}
5218
5219static bool hmR0VmxIsValidReadField(uint32_t idxField)
5220{
5221 switch (idxField)
5222 {
5223 /* Read-only fields. */
5224 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5225 return true;
5226 }
5227 /* Remaining readable fields should also be writable. */
5228 return hmR0VmxIsValidWriteField(idxField);
5229}
5230#endif /* VBOX_STRICT */
5231
5232
5233/**
5234 * Executes the specified handler in 64-bit mode.
5235 *
5236 * @returns VBox status code (no informational status codes).
5237 * @param pVCpu The cross context virtual CPU structure.
5238 * @param enmOp The operation to perform.
5239 * @param cParams Number of parameters.
5240 * @param paParam Array of 32-bit parameters.
5241 */
5242VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5243{
5244 PVM pVM = pVCpu->CTX_SUFF(pVM);
5245 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5246 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5247 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5248 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5249
5250#ifdef VBOX_STRICT
5251 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5252 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5253
5254 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5255 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5256#endif
5257
5258 /* Disable interrupts. */
5259 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5260
5261#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5262 RTCPUID idHostCpu = RTMpCpuId();
5263 CPUMR0SetLApic(pVCpu, idHostCpu);
5264#endif
5265
5266 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5267 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5268
5269 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5270 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5271 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5272
5273 /* Leave VMX Root Mode. */
5274 VMXDisable();
5275
5276 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5277
5278 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5279 CPUMSetHyperEIP(pVCpu, enmOp);
5280 for (int i = (int)cParams - 1; i >= 0; i--)
5281 CPUMPushHyper(pVCpu, paParam[i]);
5282
5283 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285 /* Call the switcher. */
5286 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5287 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5288
5289 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5290 /* Make sure the VMX instructions don't cause #UD faults. */
5291 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5292
5293 /* Re-enter VMX Root Mode */
5294 int rc2 = VMXEnable(HCPhysCpuPage);
5295 if (RT_FAILURE(rc2))
5296 {
5297 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5298 ASMSetFlags(fOldEFlags);
5299 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5300 return rc2;
5301 }
5302
5303 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5304 AssertRC(rc2);
5305 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5306 Assert(!(ASMGetFlags() & X86_EFL_IF));
5307 ASMSetFlags(fOldEFlags);
5308 return rc;
5309}
5310
5311
5312/**
5313 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5314 * supporting 64-bit guests.
5315 *
5316 * @returns VBox status code.
5317 * @param fResume Whether to VMLAUNCH or VMRESUME.
5318 * @param pCtx Pointer to the guest-CPU context.
5319 * @param pCache Pointer to the VMCS cache.
5320 * @param pVM The cross context VM structure.
5321 * @param pVCpu The cross context virtual CPU structure.
5322 */
5323DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5324{
5325 NOREF(fResume);
5326
5327 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5328 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5329
5330#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5331 pCache->uPos = 1;
5332 pCache->interPD = PGMGetInterPaeCR3(pVM);
5333 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5334#endif
5335
5336#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5337 pCache->TestIn.HCPhysCpuPage = 0;
5338 pCache->TestIn.HCPhysVmcs = 0;
5339 pCache->TestIn.pCache = 0;
5340 pCache->TestOut.HCPhysVmcs = 0;
5341 pCache->TestOut.pCache = 0;
5342 pCache->TestOut.pCtx = 0;
5343 pCache->TestOut.eflags = 0;
5344#else
5345 NOREF(pCache);
5346#endif
5347
5348 uint32_t aParam[10];
5349 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5350 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5351 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5352 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5353 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5354 aParam[5] = 0;
5355 aParam[6] = VM_RC_ADDR(pVM, pVM);
5356 aParam[7] = 0;
5357 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5358 aParam[9] = 0;
5359
5360#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5361 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5362 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5363#endif
5364 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5365
5366#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5367 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5368 Assert(pCtx->dr[4] == 10);
5369 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5370#endif
5371
5372#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5373 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5374 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5375 pVCpu->hm.s.vmx.HCPhysVmcs));
5376 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5377 pCache->TestOut.HCPhysVmcs));
5378 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5379 pCache->TestOut.pCache));
5380 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5381 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5382 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5383 pCache->TestOut.pCtx));
5384 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5385#endif
5386 NOREF(pCtx);
5387 return rc;
5388}
5389
5390
5391/**
5392 * Initialize the VMCS-Read cache.
5393 *
5394 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5395 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5396 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5397 * (those that have a 32-bit FULL & HIGH part).
5398 *
5399 * @returns VBox status code.
5400 * @param pVCpu The cross context virtual CPU structure.
5401 */
5402static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5403{
5404#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5405 do { \
5406 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5407 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5408 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5409 ++cReadFields; \
5410 } while (0)
5411
5412 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5413 uint32_t cReadFields = 0;
5414
5415 /*
5416 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5417 * and serve to indicate exceptions to the rules.
5418 */
5419
5420 /* Guest-natural selector base fields. */
5421#if 0
5422 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5425#endif
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5438#if 0
5439 /* Unused natural width guest-state fields. */
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5442#endif
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5445
5446 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5447 these 64-bit fields (using "FULL" and "HIGH" fields). */
5448#if 0
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5458#endif
5459
5460 /* Natural width guest-state fields. */
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5462#if 0
5463 /* Currently unused field. */
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5465#endif
5466
5467 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5468 {
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5470 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5471 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5472 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5473 }
5474 else
5475 {
5476 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5477 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5478 }
5479
5480#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5481 return VINF_SUCCESS;
5482}
5483
5484
5485/**
5486 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5487 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5488 * darwin, running 64-bit guests).
5489 *
5490 * @returns VBox status code.
5491 * @param pVCpu The cross context virtual CPU structure.
5492 * @param idxField The VMCS field encoding.
5493 * @param u64Val 16, 32 or 64-bit value.
5494 */
5495VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5496{
5497 int rc;
5498 switch (idxField)
5499 {
5500 /*
5501 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5502 */
5503 /* 64-bit Control fields. */
5504 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5505 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5506 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5507 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5508 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5509 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5510 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5511 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5512 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5513 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5514 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5515 case VMX_VMCS64_CTRL_EPTP_FULL:
5516 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5517 /* 64-bit Guest-state fields. */
5518 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5519 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5520 case VMX_VMCS64_GUEST_PAT_FULL:
5521 case VMX_VMCS64_GUEST_EFER_FULL:
5522 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5523 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5524 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5525 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5526 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5527 /* 64-bit Host-state fields. */
5528 case VMX_VMCS64_HOST_PAT_FULL:
5529 case VMX_VMCS64_HOST_EFER_FULL:
5530 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5531 {
5532 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5533 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5534 break;
5535 }
5536
5537 /*
5538 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5539 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5540 */
5541 /* Natural-width Guest-state fields. */
5542 case VMX_VMCS_GUEST_CR3:
5543 case VMX_VMCS_GUEST_ES_BASE:
5544 case VMX_VMCS_GUEST_CS_BASE:
5545 case VMX_VMCS_GUEST_SS_BASE:
5546 case VMX_VMCS_GUEST_DS_BASE:
5547 case VMX_VMCS_GUEST_FS_BASE:
5548 case VMX_VMCS_GUEST_GS_BASE:
5549 case VMX_VMCS_GUEST_LDTR_BASE:
5550 case VMX_VMCS_GUEST_TR_BASE:
5551 case VMX_VMCS_GUEST_GDTR_BASE:
5552 case VMX_VMCS_GUEST_IDTR_BASE:
5553 case VMX_VMCS_GUEST_RSP:
5554 case VMX_VMCS_GUEST_RIP:
5555 case VMX_VMCS_GUEST_SYSENTER_ESP:
5556 case VMX_VMCS_GUEST_SYSENTER_EIP:
5557 {
5558 if (!(RT_HI_U32(u64Val)))
5559 {
5560 /* If this field is 64-bit, VT-x will zero out the top bits. */
5561 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5562 }
5563 else
5564 {
5565 /* Assert that only the 32->64 switcher case should ever come here. */
5566 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5567 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5568 }
5569 break;
5570 }
5571
5572 default:
5573 {
5574 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5575 rc = VERR_INVALID_PARAMETER;
5576 break;
5577 }
5578 }
5579 AssertRCReturn(rc, rc);
5580 return rc;
5581}
5582
5583
5584/**
5585 * Queue up a VMWRITE by using the VMCS write cache.
5586 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5587 *
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param idxField The VMCS field encoding.
5590 * @param u64Val 16, 32 or 64-bit value.
5591 */
5592VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593{
5594 AssertPtr(pVCpu);
5595 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5596
5597 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5598 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5599
5600 /* Make sure there are no duplicates. */
5601 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5602 {
5603 if (pCache->Write.aField[i] == idxField)
5604 {
5605 pCache->Write.aFieldVal[i] = u64Val;
5606 return VINF_SUCCESS;
5607 }
5608 }
5609
5610 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5611 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5612 pCache->Write.cValidEntries++;
5613 return VINF_SUCCESS;
5614}
5615#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5616
5617
5618/**
5619 * Sets up the usage of TSC-offsetting and updates the VMCS.
5620 *
5621 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5622 * VMX preemption timer.
5623 *
5624 * @returns VBox status code.
5625 * @param pVCpu The cross context virtual CPU structure.
5626 *
5627 * @remarks No-long-jump zone!!!
5628 */
5629static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5630{
5631 bool fOffsettedTsc;
5632 bool fParavirtTsc;
5633 PVM pVM = pVCpu->CTX_SUFF(pVM);
5634 uint64_t uTscOffset;
5635 if (pVM->hm.s.vmx.fUsePreemptTimer)
5636 {
5637 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5638
5639 /* Make sure the returned values have sane upper and lower boundaries. */
5640 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5641 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5642 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5643 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5644
5645 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5646 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5647 AssertRC(rc);
5648 }
5649 else
5650 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5651
5652 /** @todo later optimize this to be done elsewhere and not before every
5653 * VM-entry. */
5654 if (fParavirtTsc)
5655 {
5656 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5657 information before every VM-entry, hence disable it for performance sake. */
5658#if 0
5659 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5660 AssertRC(rc);
5661#endif
5662 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5663 }
5664
5665 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5666 if ( fOffsettedTsc
5667 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5668 {
5669 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5670 {
5671 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5672 AssertRC(rc);
5673 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5674 }
5675
5676 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5677 {
5678 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5679 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5680 AssertRC(rc);
5681 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5682 }
5683 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5684 }
5685 else
5686 {
5687 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5688 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5689 {
5690 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5691 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5692 AssertRC(rc);
5693 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5694 }
5695 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5696 }
5697}
5698
5699
5700/**
5701 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5702 * VM-exit interruption info type.
5703 *
5704 * @returns The IEM exception flags.
5705 * @param uVector The event vector.
5706 * @param uVmxVectorType The VMX event type.
5707 *
5708 * @remarks This function currently only constructs flags required for
5709 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5710 * and CR2 aspects of an exception are not included).
5711 */
5712static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5713{
5714 uint32_t fIemXcptFlags;
5715 switch (uVmxVectorType)
5716 {
5717 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5718 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5719 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5720 break;
5721
5722 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5724 break;
5725
5726 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5727 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5728 break;
5729
5730 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5731 {
5732 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5733 if (uVector == X86_XCPT_BP)
5734 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5735 else if (uVector == X86_XCPT_OF)
5736 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5737 else
5738 {
5739 fIemXcptFlags = 0;
5740 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5741 }
5742 break;
5743 }
5744
5745 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5746 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5747 break;
5748
5749 default:
5750 fIemXcptFlags = 0;
5751 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5752 break;
5753 }
5754 return fIemXcptFlags;
5755}
5756
5757
5758/**
5759 * Sets an event as a pending event to be injected into the guest.
5760 *
5761 * @param pVCpu The cross context virtual CPU structure.
5762 * @param u32IntInfo The VM-entry interruption-information field.
5763 * @param cbInstr The VM-entry instruction length in bytes (for software
5764 * interrupts, exceptions and privileged software
5765 * exceptions).
5766 * @param u32ErrCode The VM-entry exception error code.
5767 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5768 * page-fault.
5769 *
5770 * @remarks Statistics counter assumes this is a guest event being injected or
5771 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5772 * always incremented.
5773 */
5774DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5775 RTGCUINTPTR GCPtrFaultAddress)
5776{
5777 Assert(!pVCpu->hm.s.Event.fPending);
5778 pVCpu->hm.s.Event.fPending = true;
5779 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5780 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5781 pVCpu->hm.s.Event.cbInstr = cbInstr;
5782 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5783}
5784
5785
5786/**
5787 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5788 *
5789 * @param pVCpu The cross context virtual CPU structure.
5790 */
5791DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5792{
5793 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5795 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5796 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5797}
5798
5799
5800/**
5801 * Handle a condition that occurred while delivering an event through the guest
5802 * IDT.
5803 *
5804 * @returns Strict VBox status code (i.e. informational status codes too).
5805 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5806 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5807 * to continue execution of the guest which will delivery the \#DF.
5808 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5809 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5810 *
5811 * @param pVCpu The cross context virtual CPU structure.
5812 * @param pVmxTransient Pointer to the VMX transient structure.
5813 *
5814 * @remarks No-long-jump zone!!!
5815 */
5816static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5817{
5818 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5819
5820 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5821 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5822 AssertRCReturn(rc2, rc2);
5823
5824 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5825 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5826 {
5827 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5828 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5829
5830 /*
5831 * If the event was a software interrupt (generated with INT n) or a software exception
5832 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5833 * can handle the VM-exit and continue guest execution which will re-execute the
5834 * instruction rather than re-injecting the exception, as that can cause premature
5835 * trips to ring-3 before injection and involve TRPM which currently has no way of
5836 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5837 * the problem).
5838 */
5839 IEMXCPTRAISE enmRaise;
5840 IEMXCPTRAISEINFO fRaiseInfo;
5841 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5842 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5843 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5844 {
5845 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5846 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5847 }
5848 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5849 {
5850 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5851 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5852 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5853 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5854 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5855 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5856 uExitVectorType), VERR_VMX_IPE_5);
5857
5858 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5859
5860 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5861 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5862 {
5863 pVmxTransient->fVectoringPF = true;
5864 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5865 }
5866 }
5867 else
5868 {
5869 /*
5870 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5871 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5872 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5873 */
5874 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5875 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5876 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5877 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5878 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5879 }
5880
5881 /*
5882 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5883 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5884 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5885 * subsequent VM-entry would fail.
5886 *
5887 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5888 */
5889 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5890 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5891 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5892 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5893 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5894 {
5895 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5896 }
5897
5898 switch (enmRaise)
5899 {
5900 case IEMXCPTRAISE_CURRENT_XCPT:
5901 {
5902 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5903 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5904 Assert(rcStrict == VINF_SUCCESS);
5905 break;
5906 }
5907
5908 case IEMXCPTRAISE_PREV_EVENT:
5909 {
5910 uint32_t u32ErrCode;
5911 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5912 {
5913 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5914 AssertRCReturn(rc2, rc2);
5915 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5916 }
5917 else
5918 u32ErrCode = 0;
5919
5920 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5923 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5924
5925 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5926 pVCpu->hm.s.Event.u32ErrCode));
5927 Assert(rcStrict == VINF_SUCCESS);
5928 break;
5929 }
5930
5931 case IEMXCPTRAISE_REEXEC_INSTR:
5932 Assert(rcStrict == VINF_SUCCESS);
5933 break;
5934
5935 case IEMXCPTRAISE_DOUBLE_FAULT:
5936 {
5937 /*
5938 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5939 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5940 */
5941 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5942 {
5943 pVmxTransient->fVectoringDoublePF = true;
5944 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5945 pVCpu->cpum.GstCtx.cr2));
5946 rcStrict = VINF_SUCCESS;
5947 }
5948 else
5949 {
5950 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5951 hmR0VmxSetPendingXcptDF(pVCpu);
5952 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5953 uIdtVector, uExitVector));
5954 rcStrict = VINF_HM_DOUBLE_FAULT;
5955 }
5956 break;
5957 }
5958
5959 case IEMXCPTRAISE_TRIPLE_FAULT:
5960 {
5961 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5962 rcStrict = VINF_EM_RESET;
5963 break;
5964 }
5965
5966 case IEMXCPTRAISE_CPU_HANG:
5967 {
5968 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5969 rcStrict = VERR_EM_GUEST_CPU_HANG;
5970 break;
5971 }
5972
5973 default:
5974 {
5975 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5976 rcStrict = VERR_VMX_IPE_2;
5977 break;
5978 }
5979 }
5980 }
5981 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5982 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5983 && uExitVector != X86_XCPT_DF
5984 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5985 {
5986 /*
5987 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5988 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5989 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5990 */
5991 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5992 {
5993 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5994 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5995 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5996 }
5997 }
5998
5999 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6000 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6001 return rcStrict;
6002}
6003
6004
6005/**
6006 * Imports a guest segment register from the current VMCS into
6007 * the guest-CPU context.
6008 *
6009 * @returns VBox status code.
6010 * @param pVCpu The cross context virtual CPU structure.
6011 * @param idxSel Index of the selector in the VMCS.
6012 * @param idxLimit Index of the segment limit in the VMCS.
6013 * @param idxBase Index of the segment base in the VMCS.
6014 * @param idxAccess Index of the access rights of the segment in the VMCS.
6015 * @param pSelReg Pointer to the segment selector.
6016 *
6017 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6018 * do not log!
6019 *
6020 * @remarks Never call this function directly!!! Use the
6021 * HMVMX_IMPORT_SREG() macro as that takes care
6022 * of whether to read from the VMCS cache or not.
6023 */
6024static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6025 PCPUMSELREG pSelReg)
6026{
6027 NOREF(pVCpu);
6028
6029 uint32_t u32Sel;
6030 uint32_t u32Limit;
6031 uint32_t u32Attr;
6032 uint64_t u64Base;
6033 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6034 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6035 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6036 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6037 AssertRCReturn(rc, rc);
6038
6039 pSelReg->Sel = (uint16_t)u32Sel;
6040 pSelReg->ValidSel = (uint16_t)u32Sel;
6041 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6042 pSelReg->u32Limit = u32Limit;
6043 pSelReg->u64Base = u64Base;
6044 pSelReg->Attr.u = u32Attr;
6045
6046 /*
6047 * If VT-x marks the segment as unusable, most other bits remain undefined:
6048 * - For CS the L, D and G bits have meaning.
6049 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6050 * - For the remaining data segments no bits are defined.
6051 *
6052 * The present bit and the unusable bit has been observed to be set at the
6053 * same time (the selector was supposed to be invalid as we started executing
6054 * a V8086 interrupt in ring-0).
6055 *
6056 * What should be important for the rest of the VBox code, is that the P bit is
6057 * cleared. Some of the other VBox code recognizes the unusable bit, but
6058 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6059 * safe side here, we'll strip off P and other bits we don't care about. If
6060 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6061 *
6062 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6063 */
6064 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6065 {
6066 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6067
6068 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6069 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6070 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6071#ifdef VBOX_STRICT
6072 VMMRZCallRing3Disable(pVCpu);
6073 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6074# ifdef DEBUG_bird
6075 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6076 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6077 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6078# endif
6079 VMMRZCallRing3Enable(pVCpu);
6080#endif
6081 }
6082 return VINF_SUCCESS;
6083}
6084
6085
6086/**
6087 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6088 *
6089 * @returns VBox status code.
6090 * @param pVCpu The cross context virtual CPU structure.
6091 *
6092 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6093 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6094 * instead!!!
6095 */
6096DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6097{
6098 uint64_t u64Val;
6099 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6100 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6101 {
6102 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6103 if (RT_SUCCESS(rc))
6104 {
6105 pCtx->rip = u64Val;
6106 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6107 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6108 }
6109 return rc;
6110 }
6111 return VINF_SUCCESS;
6112}
6113
6114
6115/**
6116 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6117 *
6118 * @returns VBox status code.
6119 * @param pVCpu The cross context virtual CPU structure.
6120 *
6121 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6122 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6123 * instead!!!
6124 */
6125DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6126{
6127 uint32_t u32Val;
6128 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6129 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6130 {
6131 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6132 if (RT_SUCCESS(rc))
6133 {
6134 pCtx->eflags.u32 = u32Val;
6135
6136 /* Restore eflags for real-on-v86-mode hack. */
6137 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6138 {
6139 pCtx->eflags.Bits.u1VM = 0;
6140 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6141 }
6142 }
6143 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6144 return rc;
6145 }
6146 return VINF_SUCCESS;
6147}
6148
6149
6150/**
6151 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6152 * context.
6153 *
6154 * @returns VBox status code.
6155 * @param pVCpu The cross context virtual CPU structure.
6156 *
6157 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6158 * do not log!
6159 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6160 * instead!!!
6161 */
6162DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6163{
6164 uint32_t u32Val;
6165 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6166 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6167 if (RT_SUCCESS(rc))
6168 {
6169 /*
6170 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6171 * might need them in hmR0VmxEvaluatePendingEvent().
6172 */
6173 if (!u32Val)
6174 {
6175 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6176 {
6177 rc = hmR0VmxImportGuestRip(pVCpu);
6178 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6179 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6180 }
6181
6182 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6183 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6184 }
6185 else
6186 {
6187 rc = hmR0VmxImportGuestRip(pVCpu);
6188 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6189
6190 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6191 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6192 {
6193 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6194 }
6195 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6196 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6197
6198 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6199 {
6200 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6201 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6202 }
6203 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6204 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6205 }
6206 }
6207 return rc;
6208}
6209
6210
6211/**
6212 * Worker for VMXR0ImportStateOnDemand.
6213 *
6214 * @returns VBox status code.
6215 * @param pVCpu The cross context virtual CPU structure.
6216 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6217 */
6218static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6219{
6220#define VMXLOCAL_BREAK_RC(a_rc) \
6221 if (RT_FAILURE(a_rc)) \
6222 break
6223
6224 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6225
6226 int rc = VINF_SUCCESS;
6227 PVM pVM = pVCpu->CTX_SUFF(pVM);
6228 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6229 uint64_t u64Val;
6230 uint32_t u32Val;
6231
6232 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6233
6234 /*
6235 * We disable interrupts to make the updating of the state and in particular
6236 * the fExtrn modification atomic wrt to preemption hooks.
6237 */
6238 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6239
6240 fWhat &= pCtx->fExtrn;
6241 if (fWhat)
6242 {
6243 do
6244 {
6245 if (fWhat & CPUMCTX_EXTRN_RIP)
6246 {
6247 rc = hmR0VmxImportGuestRip(pVCpu);
6248 VMXLOCAL_BREAK_RC(rc);
6249 }
6250
6251 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6252 {
6253 rc = hmR0VmxImportGuestRFlags(pVCpu);
6254 VMXLOCAL_BREAK_RC(rc);
6255 }
6256
6257 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6258 {
6259 rc = hmR0VmxImportGuestIntrState(pVCpu);
6260 VMXLOCAL_BREAK_RC(rc);
6261 }
6262
6263 if (fWhat & CPUMCTX_EXTRN_RSP)
6264 {
6265 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6266 VMXLOCAL_BREAK_RC(rc);
6267 pCtx->rsp = u64Val;
6268 }
6269
6270 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6271 {
6272 if (fWhat & CPUMCTX_EXTRN_CS)
6273 {
6274 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6275 rc |= hmR0VmxImportGuestRip(pVCpu);
6276 VMXLOCAL_BREAK_RC(rc);
6277 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6278 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6279 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6280 }
6281 if (fWhat & CPUMCTX_EXTRN_SS)
6282 {
6283 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6284 VMXLOCAL_BREAK_RC(rc);
6285 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6286 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6287 }
6288 if (fWhat & CPUMCTX_EXTRN_DS)
6289 {
6290 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6291 VMXLOCAL_BREAK_RC(rc);
6292 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6293 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6294 }
6295 if (fWhat & CPUMCTX_EXTRN_ES)
6296 {
6297 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6298 VMXLOCAL_BREAK_RC(rc);
6299 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6300 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6301 }
6302 if (fWhat & CPUMCTX_EXTRN_FS)
6303 {
6304 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6305 VMXLOCAL_BREAK_RC(rc);
6306 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6307 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6308 }
6309 if (fWhat & CPUMCTX_EXTRN_GS)
6310 {
6311 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6312 VMXLOCAL_BREAK_RC(rc);
6313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6314 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6315 }
6316 }
6317
6318 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6319 {
6320 if (fWhat & CPUMCTX_EXTRN_LDTR)
6321 {
6322 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6323 VMXLOCAL_BREAK_RC(rc);
6324 }
6325
6326 if (fWhat & CPUMCTX_EXTRN_GDTR)
6327 {
6328 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6329 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6330 VMXLOCAL_BREAK_RC(rc);
6331 pCtx->gdtr.pGdt = u64Val;
6332 pCtx->gdtr.cbGdt = u32Val;
6333 }
6334
6335 /* Guest IDTR. */
6336 if (fWhat & CPUMCTX_EXTRN_IDTR)
6337 {
6338 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6339 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6340 VMXLOCAL_BREAK_RC(rc);
6341 pCtx->idtr.pIdt = u64Val;
6342 pCtx->idtr.cbIdt = u32Val;
6343 }
6344
6345 /* Guest TR. */
6346 if (fWhat & CPUMCTX_EXTRN_TR)
6347 {
6348 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6349 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6350 {
6351 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6352 VMXLOCAL_BREAK_RC(rc);
6353 }
6354 }
6355 }
6356
6357 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6358 {
6359 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6360 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6361 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6362 pCtx->SysEnter.cs = u32Val;
6363 VMXLOCAL_BREAK_RC(rc);
6364 }
6365
6366#if HC_ARCH_BITS == 64
6367 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6368 {
6369 if ( pVM->hm.s.fAllow64BitGuests
6370 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6371 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6372 }
6373
6374 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6375 {
6376 if ( pVM->hm.s.fAllow64BitGuests
6377 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6378 {
6379 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6380 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6381 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6382 }
6383 }
6384#endif
6385
6386 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6387#if HC_ARCH_BITS == 32
6388 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6389#endif
6390 )
6391 {
6392 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6393 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6394 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6395 {
6396 switch (pMsr->u32Msr)
6397 {
6398#if HC_ARCH_BITS == 32
6399 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6400 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6401 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6402 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6403#endif
6404 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6405 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6406 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6407 default:
6408 {
6409 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6410 ASMSetFlags(fEFlags);
6411 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6412 cMsrs));
6413 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6414 }
6415 }
6416 }
6417 }
6418
6419 if (fWhat & CPUMCTX_EXTRN_DR7)
6420 {
6421 if (!pVCpu->hm.s.fUsingHyperDR7)
6422 {
6423 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6424 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6425 VMXLOCAL_BREAK_RC(rc);
6426 pCtx->dr[7] = u32Val;
6427 }
6428 }
6429
6430 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6431 {
6432 uint32_t u32Shadow;
6433 if (fWhat & CPUMCTX_EXTRN_CR0)
6434 {
6435 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6436 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6437 VMXLOCAL_BREAK_RC(rc);
6438 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6439 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6440 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6441 CPUMSetGuestCR0(pVCpu, u32Val);
6442 VMMRZCallRing3Enable(pVCpu);
6443 }
6444
6445 if (fWhat & CPUMCTX_EXTRN_CR4)
6446 {
6447 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6448 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6449 VMXLOCAL_BREAK_RC(rc);
6450 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6451 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6452 CPUMSetGuestCR4(pVCpu, u32Val);
6453 }
6454
6455 if (fWhat & CPUMCTX_EXTRN_CR3)
6456 {
6457 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6458 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6459 || ( pVM->hm.s.fNestedPaging
6460 && CPUMIsGuestPagingEnabledEx(pCtx)))
6461 {
6462 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6463 if (pCtx->cr3 != u64Val)
6464 {
6465 CPUMSetGuestCR3(pVCpu, u64Val);
6466 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6467 }
6468
6469 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6470 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6471 if (CPUMIsGuestInPAEModeEx(pCtx))
6472 {
6473 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6474 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6475 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6476 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6477 VMXLOCAL_BREAK_RC(rc);
6478 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6479 }
6480 }
6481 }
6482 }
6483 } while (0);
6484
6485 if (RT_SUCCESS(rc))
6486 {
6487 /* Update fExtrn. */
6488 pCtx->fExtrn &= ~fWhat;
6489
6490 /* If everything has been imported, clear the HM keeper bit. */
6491 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6492 {
6493 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6494 Assert(!pCtx->fExtrn);
6495 }
6496 }
6497 }
6498 else
6499 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6500
6501 ASMSetFlags(fEFlags);
6502
6503 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6504
6505 /*
6506 * Honor any pending CR3 updates.
6507 *
6508 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6509 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6510 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6511 *
6512 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6513 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6514 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6515 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6516 *
6517 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6518 */
6519 if (VMMRZCallRing3IsEnabled(pVCpu))
6520 {
6521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6522 {
6523 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6524 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6525 }
6526
6527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6528 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6529
6530 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6531 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6532 }
6533
6534 return VINF_SUCCESS;
6535#undef VMXLOCAL_BREAK_RC
6536}
6537
6538
6539/**
6540 * Saves the guest state from the VMCS into the guest-CPU context.
6541 *
6542 * @returns VBox status code.
6543 * @param pVCpu The cross context virtual CPU structure.
6544 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6545 */
6546VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6547{
6548 return hmR0VmxImportGuestState(pVCpu, fWhat);
6549}
6550
6551
6552/**
6553 * Check per-VM and per-VCPU force flag actions that require us to go back to
6554 * ring-3 for one reason or another.
6555 *
6556 * @returns Strict VBox status code (i.e. informational status codes too)
6557 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6558 * ring-3.
6559 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6560 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6561 * interrupts)
6562 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6563 * all EMTs to be in ring-3.
6564 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6565 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6566 * to the EM loop.
6567 *
6568 * @param pVCpu The cross context virtual CPU structure.
6569 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6570 */
6571static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6572{
6573 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6574
6575 /*
6576 * Anything pending? Should be more likely than not if we're doing a good job.
6577 */
6578 PVM pVM = pVCpu->CTX_SUFF(pVM);
6579 if ( !fStepping
6580 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6581 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6582 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6583 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6584 return VINF_SUCCESS;
6585
6586 /* Pending PGM C3 sync. */
6587 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6588 {
6589 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6590 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6591 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6592 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6593 if (rcStrict2 != VINF_SUCCESS)
6594 {
6595 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6596 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6597 return rcStrict2;
6598 }
6599 }
6600
6601 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6602 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6603 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6604 {
6605 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6606 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6607 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6608 return rc2;
6609 }
6610
6611 /* Pending VM request packets, such as hardware interrupts. */
6612 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6613 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6614 {
6615 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6616 return VINF_EM_PENDING_REQUEST;
6617 }
6618
6619 /* Pending PGM pool flushes. */
6620 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6621 {
6622 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6623 return VINF_PGM_POOL_FLUSH_PENDING;
6624 }
6625
6626 /* Pending DMA requests. */
6627 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6628 {
6629 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6630 return VINF_EM_RAW_TO_R3;
6631 }
6632
6633 return VINF_SUCCESS;
6634}
6635
6636
6637/**
6638 * Converts any TRPM trap into a pending HM event. This is typically used when
6639 * entering from ring-3 (not longjmp returns).
6640 *
6641 * @param pVCpu The cross context virtual CPU structure.
6642 */
6643static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6644{
6645 Assert(TRPMHasTrap(pVCpu));
6646 Assert(!pVCpu->hm.s.Event.fPending);
6647
6648 uint8_t uVector;
6649 TRPMEVENT enmTrpmEvent;
6650 RTGCUINT uErrCode;
6651 RTGCUINTPTR GCPtrFaultAddress;
6652 uint8_t cbInstr;
6653
6654 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6655 AssertRC(rc);
6656
6657 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6658 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6659 if (enmTrpmEvent == TRPM_TRAP)
6660 {
6661 switch (uVector)
6662 {
6663 case X86_XCPT_NMI:
6664 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6665 break;
6666
6667 case X86_XCPT_BP:
6668 case X86_XCPT_OF:
6669 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6670 break;
6671
6672 case X86_XCPT_PF:
6673 case X86_XCPT_DF:
6674 case X86_XCPT_TS:
6675 case X86_XCPT_NP:
6676 case X86_XCPT_SS:
6677 case X86_XCPT_GP:
6678 case X86_XCPT_AC:
6679 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6680 RT_FALL_THRU();
6681 default:
6682 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6683 break;
6684 }
6685 }
6686 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6688 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6689 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6690 else
6691 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6692
6693 rc = TRPMResetTrap(pVCpu);
6694 AssertRC(rc);
6695 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6696 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6697
6698 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6699}
6700
6701
6702/**
6703 * Converts the pending HM event into a TRPM trap.
6704 *
6705 * @param pVCpu The cross context virtual CPU structure.
6706 */
6707static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6708{
6709 Assert(pVCpu->hm.s.Event.fPending);
6710
6711 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6712 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6713 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6714 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6715
6716 /* If a trap was already pending, we did something wrong! */
6717 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6718
6719 TRPMEVENT enmTrapType;
6720 switch (uVectorType)
6721 {
6722 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6723 enmTrapType = TRPM_HARDWARE_INT;
6724 break;
6725
6726 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6727 enmTrapType = TRPM_SOFTWARE_INT;
6728 break;
6729
6730 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6731 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6732 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6733 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6734 enmTrapType = TRPM_TRAP;
6735 break;
6736
6737 default:
6738 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6739 enmTrapType = TRPM_32BIT_HACK;
6740 break;
6741 }
6742
6743 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6744
6745 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6746 AssertRC(rc);
6747
6748 if (fErrorCodeValid)
6749 TRPMSetErrorCode(pVCpu, uErrorCode);
6750
6751 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6752 && uVector == X86_XCPT_PF)
6753 {
6754 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6755 }
6756 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6757 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6758 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6759 {
6760 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6761 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6762 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6763 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6764 }
6765
6766 /* Clear any pending events from the VMCS. */
6767 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6768 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6769
6770 /* We're now done converting the pending event. */
6771 pVCpu->hm.s.Event.fPending = false;
6772}
6773
6774
6775/**
6776 * Does the necessary state syncing before returning to ring-3 for any reason
6777 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6778 *
6779 * @returns VBox status code.
6780 * @param pVCpu The cross context virtual CPU structure.
6781 * @param fImportState Whether to import the guest state from the VMCS back
6782 * to the guest-CPU context.
6783 *
6784 * @remarks No-long-jmp zone!!!
6785 */
6786static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6787{
6788 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6789 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6790
6791 RTCPUID idCpu = RTMpCpuId();
6792 Log4Func(("HostCpuId=%u\n", idCpu));
6793
6794 /*
6795 * !!! IMPORTANT !!!
6796 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6797 */
6798
6799 /* Save the guest state if necessary. */
6800 if (fImportState)
6801 {
6802 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6803 AssertRCReturn(rc, rc);
6804 }
6805
6806 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6807 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6808 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6809
6810 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6811#ifdef VBOX_STRICT
6812 if (CPUMIsHyperDebugStateActive(pVCpu))
6813 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6814#endif
6815 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6816 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6817 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6818
6819#if HC_ARCH_BITS == 64
6820 /* Restore host-state bits that VT-x only restores partially. */
6821 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6822 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6823 {
6824 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6825 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6826 }
6827 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6828#endif
6829
6830 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6831 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6832 {
6833 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6834 if (!fImportState)
6835 {
6836 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6837 AssertRCReturn(rc, rc);
6838 }
6839 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6840 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6841 }
6842 else
6843 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6844
6845 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6846 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6847
6848 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6849 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6850 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6851 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6852 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6853 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6854 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6855 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6856 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6857
6858 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6859
6860 /** @todo This partially defeats the purpose of having preemption hooks.
6861 * The problem is, deregistering the hooks should be moved to a place that
6862 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6863 * context.
6864 */
6865 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6866 {
6867 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6868 AssertRCReturn(rc, rc);
6869
6870 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6871 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6872 }
6873 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6874 NOREF(idCpu);
6875
6876 return VINF_SUCCESS;
6877}
6878
6879
6880/**
6881 * Leaves the VT-x session.
6882 *
6883 * @returns VBox status code.
6884 * @param pVCpu The cross context virtual CPU structure.
6885 *
6886 * @remarks No-long-jmp zone!!!
6887 */
6888static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6889{
6890 HM_DISABLE_PREEMPT(pVCpu);
6891 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6892 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6893 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6894
6895 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6896 and done this from the VMXR0ThreadCtxCallback(). */
6897 if (!pVCpu->hm.s.fLeaveDone)
6898 {
6899 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6900 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6901 pVCpu->hm.s.fLeaveDone = true;
6902 }
6903 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6904
6905 /*
6906 * !!! IMPORTANT !!!
6907 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6908 */
6909
6910 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6911 /** @todo Deregistering here means we need to VMCLEAR always
6912 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6913 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6914 VMMR0ThreadCtxHookDisable(pVCpu);
6915
6916 /* Leave HM context. This takes care of local init (term). */
6917 int rc = HMR0LeaveCpu(pVCpu);
6918
6919 HM_RESTORE_PREEMPT();
6920 return rc;
6921}
6922
6923
6924/**
6925 * Does the necessary state syncing before doing a longjmp to ring-3.
6926 *
6927 * @returns VBox status code.
6928 * @param pVCpu The cross context virtual CPU structure.
6929 *
6930 * @remarks No-long-jmp zone!!!
6931 */
6932DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6933{
6934 return hmR0VmxLeaveSession(pVCpu);
6935}
6936
6937
6938/**
6939 * Take necessary actions before going back to ring-3.
6940 *
6941 * An action requires us to go back to ring-3. This function does the necessary
6942 * steps before we can safely return to ring-3. This is not the same as longjmps
6943 * to ring-3, this is voluntary and prepares the guest so it may continue
6944 * executing outside HM (recompiler/IEM).
6945 *
6946 * @returns VBox status code.
6947 * @param pVCpu The cross context virtual CPU structure.
6948 * @param rcExit The reason for exiting to ring-3. Can be
6949 * VINF_VMM_UNKNOWN_RING3_CALL.
6950 */
6951static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6952{
6953 Assert(pVCpu);
6954 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6955
6956 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6957 {
6958 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6959 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6960 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6961 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6962 }
6963
6964 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6965 VMMRZCallRing3Disable(pVCpu);
6966 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6967
6968 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6969 if (pVCpu->hm.s.Event.fPending)
6970 {
6971 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6972 Assert(!pVCpu->hm.s.Event.fPending);
6973 }
6974
6975 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6976 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6977
6978 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6979 and if we're injecting an event we should have a TRPM trap pending. */
6980 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6981#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6982 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6983#endif
6984
6985 /* Save guest state and restore host state bits. */
6986 int rc = hmR0VmxLeaveSession(pVCpu);
6987 AssertRCReturn(rc, rc);
6988 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6989 /* Thread-context hooks are unregistered at this point!!! */
6990
6991 /* Sync recompiler state. */
6992 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6993 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6994 | CPUM_CHANGED_LDTR
6995 | CPUM_CHANGED_GDTR
6996 | CPUM_CHANGED_IDTR
6997 | CPUM_CHANGED_TR
6998 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6999 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7000 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7001 {
7002 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7003 }
7004
7005 Assert(!pVCpu->hm.s.fClearTrapFlag);
7006
7007 /* Update the exit-to-ring 3 reason. */
7008 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7009
7010 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7011 if (rcExit != VINF_EM_RAW_INTERRUPT)
7012 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7013
7014 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7015
7016 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7017 VMMRZCallRing3RemoveNotification(pVCpu);
7018 VMMRZCallRing3Enable(pVCpu);
7019
7020 return rc;
7021}
7022
7023
7024/**
7025 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7026 * longjump to ring-3 and possibly get preempted.
7027 *
7028 * @returns VBox status code.
7029 * @param pVCpu The cross context virtual CPU structure.
7030 * @param enmOperation The operation causing the ring-3 longjump.
7031 * @param pvUser User argument, currently unused, NULL.
7032 */
7033static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7034{
7035 RT_NOREF(pvUser);
7036 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7037 {
7038 /*
7039 * !!! IMPORTANT !!!
7040 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7041 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7042 */
7043 VMMRZCallRing3RemoveNotification(pVCpu);
7044 VMMRZCallRing3Disable(pVCpu);
7045 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7046 RTThreadPreemptDisable(&PreemptState);
7047
7048 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7049 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7050 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7051
7052#if HC_ARCH_BITS == 64
7053 /* Restore host-state bits that VT-x only restores partially. */
7054 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7055 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7056 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7057 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7058#endif
7059
7060 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7061 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7062 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7063
7064 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7065 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7066 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7067 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7068 {
7069 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7070 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7071 }
7072
7073 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7074 VMMR0ThreadCtxHookDisable(pVCpu);
7075 HMR0LeaveCpu(pVCpu);
7076 RTThreadPreemptRestore(&PreemptState);
7077 return VINF_SUCCESS;
7078 }
7079
7080 Assert(pVCpu);
7081 Assert(pvUser);
7082 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7083 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7084
7085 VMMRZCallRing3Disable(pVCpu);
7086 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7087
7088 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7089
7090 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7091 AssertRCReturn(rc, rc);
7092
7093 VMMRZCallRing3Enable(pVCpu);
7094 return VINF_SUCCESS;
7095}
7096
7097
7098/**
7099 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7100 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7101 *
7102 * @param pVCpu The cross context virtual CPU structure.
7103 */
7104DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7105{
7106 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7107 {
7108 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7109 {
7110 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7111 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7112 AssertRC(rc);
7113 Log4Func(("Setup interrupt-window exiting\n"));
7114 }
7115 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7116}
7117
7118
7119/**
7120 * Clears the interrupt-window exiting control in the VMCS.
7121 *
7122 * @param pVCpu The cross context virtual CPU structure.
7123 */
7124DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7125{
7126 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7127 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7128 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7129 AssertRC(rc);
7130 Log4Func(("Cleared interrupt-window exiting\n"));
7131}
7132
7133
7134/**
7135 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7136 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7137 *
7138 * @param pVCpu The cross context virtual CPU structure.
7139 */
7140DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7141{
7142 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7143 {
7144 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7145 {
7146 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7147 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7148 AssertRC(rc);
7149 Log4Func(("Setup NMI-window exiting\n"));
7150 }
7151 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7152}
7153
7154
7155/**
7156 * Clears the NMI-window exiting control in the VMCS.
7157 *
7158 * @param pVCpu The cross context virtual CPU structure.
7159 */
7160DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7161{
7162 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7163 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7164 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7165 AssertRC(rc);
7166 Log4Func(("Cleared NMI-window exiting\n"));
7167}
7168
7169
7170/**
7171 * Evaluates the event to be delivered to the guest and sets it as the pending
7172 * event.
7173 *
7174 * @returns The VT-x guest-interruptibility state.
7175 * @param pVCpu The cross context virtual CPU structure.
7176 */
7177static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7178{
7179 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7180 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7181 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7182 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7183 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7184 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7185
7186 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7187 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7188 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7189 Assert(!TRPMHasTrap(pVCpu));
7190
7191 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7192 APICUpdatePendingInterrupts(pVCpu);
7193
7194 /*
7195 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7196 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7197 */
7198 /** @todo SMI. SMIs take priority over NMIs. */
7199 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7200 {
7201 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7202 if ( !pVCpu->hm.s.Event.fPending
7203 && !fBlockNmi
7204 && !fBlockSti
7205 && !fBlockMovSS)
7206 {
7207 Log4Func(("Pending NMI\n"));
7208 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7209 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7210
7211 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7212 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7213 }
7214 else
7215 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7216 }
7217 /*
7218 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7219 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7220 */
7221 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7222 && !pVCpu->hm.s.fSingleInstruction)
7223 {
7224 Assert(!DBGFIsStepping(pVCpu));
7225 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7226 AssertRCReturn(rc, 0);
7227 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7228 if ( !pVCpu->hm.s.Event.fPending
7229 && !fBlockInt
7230 && !fBlockSti
7231 && !fBlockMovSS)
7232 {
7233 uint8_t u8Interrupt;
7234 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7235 if (RT_SUCCESS(rc))
7236 {
7237 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7238 uint32_t u32IntInfo = u8Interrupt
7239 | VMX_EXIT_INTERRUPTION_INFO_VALID
7240 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7241
7242 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7243 }
7244 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7245 {
7246 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7247 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7248 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7249
7250 /*
7251 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7252 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7253 * need to re-set this force-flag here.
7254 */
7255 }
7256 else
7257 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7258 }
7259 else
7260 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7261 }
7262
7263 return fIntrState;
7264}
7265
7266
7267/**
7268 * Sets a pending-debug exception to be delivered to the guest if the guest is
7269 * single-stepping in the VMCS.
7270 *
7271 * @param pVCpu The cross context virtual CPU structure.
7272 */
7273DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7274{
7275 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7276 RT_NOREF(pVCpu);
7277 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7278}
7279
7280
7281/**
7282 * Injects any pending events into the guest if the guest is in a state to
7283 * receive them.
7284 *
7285 * @returns Strict VBox status code (i.e. informational status codes too).
7286 * @param pVCpu The cross context virtual CPU structure.
7287 * @param fIntrState The VT-x guest-interruptibility state.
7288 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7289 * return VINF_EM_DBG_STEPPED if the event was
7290 * dispatched directly.
7291 */
7292static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7293{
7294 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7295 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7296
7297 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7298 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7299
7300 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7301 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7302 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7303 Assert(!TRPMHasTrap(pVCpu));
7304
7305 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7306 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7307 if (pVCpu->hm.s.Event.fPending)
7308 {
7309 /*
7310 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7311 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7312 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7313 *
7314 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7315 */
7316 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7317#ifdef VBOX_STRICT
7318 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7319 {
7320 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7321 Assert(!fBlockInt);
7322 Assert(!fBlockSti);
7323 Assert(!fBlockMovSS);
7324 }
7325 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7326 {
7327 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7328 Assert(!fBlockSti);
7329 Assert(!fBlockMovSS);
7330 Assert(!fBlockNmi);
7331 }
7332#endif
7333 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7334 uIntType));
7335 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7336 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7337 &fIntrState);
7338 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7339
7340 /* Update the interruptibility-state as it could have been changed by
7341 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7342 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7343 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7344
7345 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7346 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7347 else
7348 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7349 }
7350
7351 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7352 if ( fBlockSti
7353 || fBlockMovSS)
7354 {
7355 if (!pVCpu->hm.s.fSingleInstruction)
7356 {
7357 /*
7358 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7359 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7360 * See Intel spec. 27.3.4 "Saving Non-Register State".
7361 */
7362 Assert(!DBGFIsStepping(pVCpu));
7363 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7364 AssertRCReturn(rc, rc);
7365 if (pCtx->eflags.Bits.u1TF)
7366 {
7367 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7368 AssertRCReturn(rc2, rc2);
7369 }
7370 }
7371 else if (pCtx->eflags.Bits.u1TF)
7372 {
7373 /*
7374 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7375 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7376 */
7377 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7378 fIntrState = 0;
7379 }
7380 }
7381
7382 /*
7383 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7384 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7385 */
7386 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7387 AssertRCReturn(rc3, rc3);
7388
7389 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7390 NOREF(fBlockMovSS); NOREF(fBlockSti);
7391 return rcStrict;
7392}
7393
7394
7395/**
7396 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7397 *
7398 * @param pVCpu The cross context virtual CPU structure.
7399 */
7400DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7401{
7402 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7403 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7404}
7405
7406
7407/**
7408 * Injects a double-fault (\#DF) exception into the VM.
7409 *
7410 * @returns Strict VBox status code (i.e. informational status codes too).
7411 * @param pVCpu The cross context virtual CPU structure.
7412 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7413 * and should return VINF_EM_DBG_STEPPED if the event
7414 * is injected directly (register modified by us, not
7415 * by hardware on VM-entry).
7416 * @param pfIntrState Pointer to the current guest interruptibility-state.
7417 * This interruptibility-state will be updated if
7418 * necessary. This cannot not be NULL.
7419 */
7420DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7421{
7422 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7423 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7424 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7425 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7426 pfIntrState);
7427}
7428
7429
7430/**
7431 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7432 *
7433 * @param pVCpu The cross context virtual CPU structure.
7434 */
7435DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7436{
7437 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7438 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7439 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7440}
7441
7442
7443/**
7444 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7445 *
7446 * @param pVCpu The cross context virtual CPU structure.
7447 * @param cbInstr The value of RIP that is to be pushed on the guest
7448 * stack.
7449 */
7450DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7451{
7452 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7453 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7454 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7455}
7456
7457
7458/**
7459 * Injects a general-protection (\#GP) fault into the VM.
7460 *
7461 * @returns Strict VBox status code (i.e. informational status codes too).
7462 * @param pVCpu The cross context virtual CPU structure.
7463 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7464 * mode, i.e. in real-mode it's not valid).
7465 * @param u32ErrorCode The error code associated with the \#GP.
7466 * @param fStepping Whether we're running in
7467 * hmR0VmxRunGuestCodeStep() and should return
7468 * VINF_EM_DBG_STEPPED if the event is injected
7469 * directly (register modified by us, not by
7470 * hardware on VM-entry).
7471 * @param pfIntrState Pointer to the current guest interruptibility-state.
7472 * This interruptibility-state will be updated if
7473 * necessary. This cannot not be NULL.
7474 */
7475DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7476 uint32_t *pfIntrState)
7477{
7478 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7479 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7480 if (fErrorCodeValid)
7481 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7482 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7483 pfIntrState);
7484}
7485
7486
7487/**
7488 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7489 *
7490 * @param pVCpu The cross context virtual CPU structure.
7491 * @param uVector The software interrupt vector number.
7492 * @param cbInstr The value of RIP that is to be pushed on the guest
7493 * stack.
7494 */
7495DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7496{
7497 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7498 if ( uVector == X86_XCPT_BP
7499 || uVector == X86_XCPT_OF)
7500 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7501 else
7502 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7503 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7504}
7505
7506
7507/**
7508 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7509 * stack.
7510 *
7511 * @returns Strict VBox status code (i.e. informational status codes too).
7512 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7513 * @param pVCpu The cross context virtual CPU structure.
7514 * @param uValue The value to push to the guest stack.
7515 */
7516static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7517{
7518 /*
7519 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7520 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7521 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7522 */
7523 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7524 if (pCtx->sp == 1)
7525 return VINF_EM_RESET;
7526 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7527 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7528 AssertRC(rc);
7529 return rc;
7530}
7531
7532
7533/**
7534 * Injects an event into the guest upon VM-entry by updating the relevant fields
7535 * in the VM-entry area in the VMCS.
7536 *
7537 * @returns Strict VBox status code (i.e. informational status codes too).
7538 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7539 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7540 *
7541 * @param pVCpu The cross context virtual CPU structure.
7542 * @param u64IntInfo The VM-entry interruption-information field.
7543 * @param cbInstr The VM-entry instruction length in bytes (for
7544 * software interrupts, exceptions and privileged
7545 * software exceptions).
7546 * @param u32ErrCode The VM-entry exception error code.
7547 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7548 * @param pfIntrState Pointer to the current guest interruptibility-state.
7549 * This interruptibility-state will be updated if
7550 * necessary. This cannot not be NULL.
7551 * @param fStepping Whether we're running in
7552 * hmR0VmxRunGuestCodeStep() and should return
7553 * VINF_EM_DBG_STEPPED if the event is injected
7554 * directly (register modified by us, not by
7555 * hardware on VM-entry).
7556 */
7557static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7558 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7559{
7560 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7561 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7562 Assert(pfIntrState);
7563
7564 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7565 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7566 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7567 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7568
7569#ifdef VBOX_STRICT
7570 /*
7571 * Validate the error-code-valid bit for hardware exceptions.
7572 * No error codes for exceptions in real-mode.
7573 *
7574 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7575 */
7576 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7577 && !CPUMIsGuestInRealModeEx(pCtx))
7578 {
7579 switch (uVector)
7580 {
7581 case X86_XCPT_PF:
7582 case X86_XCPT_DF:
7583 case X86_XCPT_TS:
7584 case X86_XCPT_NP:
7585 case X86_XCPT_SS:
7586 case X86_XCPT_GP:
7587 case X86_XCPT_AC:
7588 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7589 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7590 RT_FALL_THRU();
7591 default:
7592 break;
7593 }
7594 }
7595#endif
7596
7597 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7598 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7599 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7600
7601 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7602
7603 /*
7604 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7605 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7606 * interrupt handler in the (real-mode) guest.
7607 *
7608 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7609 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7610 */
7611 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7612 {
7613 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7614 {
7615 /*
7616 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7617 * set the deliver-error-code bit.
7618 *
7619 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7620 */
7621 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7622 }
7623 else
7624 {
7625 PVM pVM = pVCpu->CTX_SUFF(pVM);
7626 Assert(PDMVmmDevHeapIsEnabled(pVM));
7627 Assert(pVM->hm.s.vmx.pRealModeTSS);
7628
7629 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7630 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7631 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7632 AssertRCReturn(rc2, rc2);
7633
7634 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7635 size_t const cbIdtEntry = sizeof(X86IDTR16);
7636 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7637 {
7638 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7639 if (uVector == X86_XCPT_DF)
7640 return VINF_EM_RESET;
7641
7642 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7643 if (uVector == X86_XCPT_GP)
7644 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7645
7646 /*
7647 * If we're injecting an event with no valid IDT entry, inject a #GP.
7648 * No error codes for exceptions in real-mode.
7649 *
7650 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7651 */
7652 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7653 }
7654
7655 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7656 uint16_t uGuestIp = pCtx->ip;
7657 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7658 {
7659 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7660 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7661 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7662 }
7663 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7664 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7665
7666 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7667 X86IDTR16 IdtEntry;
7668 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7669 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7670 AssertRCReturn(rc2, rc2);
7671
7672 /* Construct the stack frame for the interrupt/exception handler. */
7673 VBOXSTRICTRC rcStrict;
7674 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7675 if (rcStrict == VINF_SUCCESS)
7676 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7677 if (rcStrict == VINF_SUCCESS)
7678 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7679
7680 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7681 if (rcStrict == VINF_SUCCESS)
7682 {
7683 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7684 pCtx->rip = IdtEntry.offSel;
7685 pCtx->cs.Sel = IdtEntry.uSel;
7686 pCtx->cs.ValidSel = IdtEntry.uSel;
7687 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7688 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7689 && uVector == X86_XCPT_PF)
7690 pCtx->cr2 = GCPtrFaultAddress;
7691
7692 /* If any other guest-state bits are changed here, make sure to update
7693 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7694 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7695 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7696 | HM_CHANGED_GUEST_RSP);
7697
7698 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7699 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7700 {
7701 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7702 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7703 Log4Func(("Clearing inhibition due to STI\n"));
7704 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7705 }
7706 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7707 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7708
7709 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7710 it, if we are returning to ring-3 before executing guest code. */
7711 pVCpu->hm.s.Event.fPending = false;
7712
7713 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7714 if (fStepping)
7715 rcStrict = VINF_EM_DBG_STEPPED;
7716 }
7717 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7718 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7719 return rcStrict;
7720 }
7721 }
7722
7723 /* Validate. */
7724 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7725 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7726 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7727
7728 /* Inject. */
7729 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7730 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7731 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7732 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7733 AssertRCReturn(rc, rc);
7734
7735 /* Update CR2. */
7736 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7737 && uVector == X86_XCPT_PF)
7738 pCtx->cr2 = GCPtrFaultAddress;
7739
7740 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7741
7742 return VINF_SUCCESS;
7743}
7744
7745
7746/**
7747 * Clears the interrupt-window exiting control in the VMCS and if necessary
7748 * clears the current event in the VMCS as well.
7749 *
7750 * @returns VBox status code.
7751 * @param pVCpu The cross context virtual CPU structure.
7752 *
7753 * @remarks Use this function only to clear events that have not yet been
7754 * delivered to the guest but are injected in the VMCS!
7755 * @remarks No-long-jump zone!!!
7756 */
7757static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7758{
7759 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7760 {
7761 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7762 Log4Func(("Cleared interrupt widow\n"));
7763 }
7764
7765 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7766 {
7767 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7768 Log4Func(("Cleared interrupt widow\n"));
7769 }
7770}
7771
7772
7773/**
7774 * Enters the VT-x session.
7775 *
7776 * @returns VBox status code.
7777 * @param pVCpu The cross context virtual CPU structure.
7778 * @param pHostCpu Pointer to the global CPU info struct.
7779 */
7780VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7781{
7782 AssertPtr(pVCpu);
7783 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7784 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7785 RT_NOREF(pHostCpu);
7786
7787 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7788 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7789 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7790
7791#ifdef VBOX_STRICT
7792 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7793 RTCCUINTREG uHostCR4 = ASMGetCR4();
7794 if (!(uHostCR4 & X86_CR4_VMXE))
7795 {
7796 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7797 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7798 }
7799#endif
7800
7801 /*
7802 * Load the VCPU's VMCS as the current (and active) one.
7803 */
7804 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7805 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7806 if (RT_FAILURE(rc))
7807 return rc;
7808
7809 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7810 pVCpu->hm.s.fLeaveDone = false;
7811 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7812
7813 return VINF_SUCCESS;
7814}
7815
7816
7817/**
7818 * The thread-context callback (only on platforms which support it).
7819 *
7820 * @param enmEvent The thread-context event.
7821 * @param pVCpu The cross context virtual CPU structure.
7822 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7823 * @thread EMT(pVCpu)
7824 */
7825VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7826{
7827 NOREF(fGlobalInit);
7828
7829 switch (enmEvent)
7830 {
7831 case RTTHREADCTXEVENT_OUT:
7832 {
7833 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7834 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7835 VMCPU_ASSERT_EMT(pVCpu);
7836
7837 /* No longjmps (logger flushes, locks) in this fragile context. */
7838 VMMRZCallRing3Disable(pVCpu);
7839 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7840
7841 /*
7842 * Restore host-state (FPU, debug etc.)
7843 */
7844 if (!pVCpu->hm.s.fLeaveDone)
7845 {
7846 /*
7847 * Do -not- import the guest-state here as we might already be in the middle of importing
7848 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7849 */
7850 hmR0VmxLeave(pVCpu, false /* fImportState */);
7851 pVCpu->hm.s.fLeaveDone = true;
7852 }
7853
7854 /* Leave HM context, takes care of local init (term). */
7855 int rc = HMR0LeaveCpu(pVCpu);
7856 AssertRC(rc); NOREF(rc);
7857
7858 /* Restore longjmp state. */
7859 VMMRZCallRing3Enable(pVCpu);
7860 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7861 break;
7862 }
7863
7864 case RTTHREADCTXEVENT_IN:
7865 {
7866 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7867 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7868 VMCPU_ASSERT_EMT(pVCpu);
7869
7870 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7871 VMMRZCallRing3Disable(pVCpu);
7872 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7873
7874 /* Initialize the bare minimum state required for HM. This takes care of
7875 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7876 int rc = hmR0EnterCpu(pVCpu);
7877 AssertRC(rc);
7878 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7879 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7880
7881 /* Load the active VMCS as the current one. */
7882 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7883 {
7884 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7885 AssertRC(rc); NOREF(rc);
7886 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7887 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7888 }
7889 pVCpu->hm.s.fLeaveDone = false;
7890
7891 /* Restore longjmp state. */
7892 VMMRZCallRing3Enable(pVCpu);
7893 break;
7894 }
7895
7896 default:
7897 break;
7898 }
7899}
7900
7901
7902/**
7903 * Exports the host state into the VMCS host-state area.
7904 * Sets up the VM-exit MSR-load area.
7905 *
7906 * The CPU state will be loaded from these fields on every successful VM-exit.
7907 *
7908 * @returns VBox status code.
7909 * @param pVCpu The cross context virtual CPU structure.
7910 *
7911 * @remarks No-long-jump zone!!!
7912 */
7913static int hmR0VmxExportHostState(PVMCPU pVCpu)
7914{
7915 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7916
7917 int rc = VINF_SUCCESS;
7918 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7919 {
7920 rc = hmR0VmxExportHostControlRegs();
7921 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7922
7923 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7924 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7925
7926 rc = hmR0VmxExportHostMsrs(pVCpu);
7927 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7928
7929 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7930 }
7931 return rc;
7932}
7933
7934
7935/**
7936 * Saves the host state in the VMCS host-state.
7937 *
7938 * @returns VBox status code.
7939 * @param pVCpu The cross context virtual CPU structure.
7940 *
7941 * @remarks No-long-jump zone!!!
7942 */
7943VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7944{
7945 AssertPtr(pVCpu);
7946 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7947
7948 /*
7949 * Export the host state here while entering HM context.
7950 * When thread-context hooks are used, we might get preempted and have to re-save the host
7951 * state but most of the time we won't be, so do it here before we disable interrupts.
7952 */
7953 return hmR0VmxExportHostState(pVCpu);
7954}
7955
7956
7957/**
7958 * Exports the guest state into the VMCS guest-state area.
7959 *
7960 * The will typically be done before VM-entry when the guest-CPU state and the
7961 * VMCS state may potentially be out of sync.
7962 *
7963 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7964 * VM-entry controls.
7965 * Sets up the appropriate VMX non-root function to execute guest code based on
7966 * the guest CPU mode.
7967 *
7968 * @returns VBox strict status code.
7969 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7970 * without unrestricted guest access and the VMMDev is not presently
7971 * mapped (e.g. EFI32).
7972 *
7973 * @param pVCpu The cross context virtual CPU structure.
7974 *
7975 * @remarks No-long-jump zone!!!
7976 */
7977static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7978{
7979 AssertPtr(pVCpu);
7980 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7981
7982 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7983
7984 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7985
7986 /* Determine real-on-v86 mode. */
7987 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7988 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7989 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7990 {
7991 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7992 }
7993
7994 /*
7995 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7996 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7997 */
7998 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7999 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8000
8001 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8002 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8003 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8004
8005 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8006 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8007 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8008
8009 rc = hmR0VmxExportGuestCR0(pVCpu);
8010 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8011
8012 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8013 if (rcStrict == VINF_SUCCESS)
8014 { /* likely */ }
8015 else
8016 {
8017 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8018 return rcStrict;
8019 }
8020
8021 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8022 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8023
8024 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8025 may alter controls if we determine we don't have to swap EFER after all. */
8026 rc = hmR0VmxExportGuestMsrs(pVCpu);
8027 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8028
8029 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8030 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8031
8032 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8033 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8034
8035 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8036 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8037 rc = hmR0VmxExportGuestRip(pVCpu);
8038 rc |= hmR0VmxExportGuestRsp(pVCpu);
8039 rc |= hmR0VmxExportGuestRflags(pVCpu);
8040 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8041
8042 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8043 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8044 | HM_CHANGED_GUEST_CR2
8045 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8046 | HM_CHANGED_GUEST_X87
8047 | HM_CHANGED_GUEST_SSE_AVX
8048 | HM_CHANGED_GUEST_OTHER_XSAVE
8049 | HM_CHANGED_GUEST_XCRx
8050 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8051 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8052 | HM_CHANGED_GUEST_TSC_AUX
8053 | HM_CHANGED_GUEST_OTHER_MSRS
8054 | HM_CHANGED_GUEST_HWVIRT
8055 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8056
8057 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8058 return rc;
8059}
8060
8061
8062/**
8063 * Exports the state shared between the host and guest into the VMCS.
8064 *
8065 * @param pVCpu The cross context virtual CPU structure.
8066 *
8067 * @remarks No-long-jump zone!!!
8068 */
8069static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8070{
8071 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8072 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8073
8074 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8075 {
8076 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8077 AssertRC(rc);
8078 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8079
8080 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8081 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8082 {
8083 rc = hmR0VmxExportGuestRflags(pVCpu);
8084 AssertRC(rc);
8085 }
8086 }
8087
8088 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8089 {
8090 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8091 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8092 }
8093
8094 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8095 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8096}
8097
8098
8099/**
8100 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8101 *
8102 * @returns Strict VBox status code (i.e. informational status codes too).
8103 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8104 * without unrestricted guest access and the VMMDev is not presently
8105 * mapped (e.g. EFI32).
8106 *
8107 * @param pVCpu The cross context virtual CPU structure.
8108 *
8109 * @remarks No-long-jump zone!!!
8110 */
8111static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8112{
8113 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8114 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8115 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8116
8117#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8118 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8119#endif
8120
8121 /*
8122 * For many exits it's only RIP that changes and hence try to export it first
8123 * without going through a lot of change flag checks.
8124 */
8125 VBOXSTRICTRC rcStrict;
8126 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8127 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8128 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8129 {
8130 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8131 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8132 { /* likely */}
8133 else
8134 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8135 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8136 }
8137 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8138 {
8139 rcStrict = hmR0VmxExportGuestState(pVCpu);
8140 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8141 { /* likely */}
8142 else
8143 {
8144 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8145 VBOXSTRICTRC_VAL(rcStrict)));
8146 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8147 return rcStrict;
8148 }
8149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8150 }
8151 else
8152 rcStrict = VINF_SUCCESS;
8153
8154#ifdef VBOX_STRICT
8155 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8156 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8157 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8158 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8159 ("fCtxChanged=%#RX64\n", fCtxChanged));
8160#endif
8161 return rcStrict;
8162}
8163
8164
8165/**
8166 * Does the preparations before executing guest code in VT-x.
8167 *
8168 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8169 * recompiler/IEM. We must be cautious what we do here regarding committing
8170 * guest-state information into the VMCS assuming we assuredly execute the
8171 * guest in VT-x mode.
8172 *
8173 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8174 * the common-state (TRPM/forceflags), we must undo those changes so that the
8175 * recompiler/IEM can (and should) use them when it resumes guest execution.
8176 * Otherwise such operations must be done when we can no longer exit to ring-3.
8177 *
8178 * @returns Strict VBox status code (i.e. informational status codes too).
8179 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8180 * have been disabled.
8181 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8182 * double-fault into the guest.
8183 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8184 * dispatched directly.
8185 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8186 *
8187 * @param pVCpu The cross context virtual CPU structure.
8188 * @param pVmxTransient Pointer to the VMX transient structure.
8189 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8190 * us ignore some of the reasons for returning to
8191 * ring-3, and return VINF_EM_DBG_STEPPED if event
8192 * dispatching took place.
8193 */
8194static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8195{
8196 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8197
8198#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8199 PGMRZDynMapFlushAutoSet(pVCpu);
8200#endif
8201
8202 /* Check force flag actions that might require us to go back to ring-3. */
8203 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8204 if (rcStrict == VINF_SUCCESS)
8205 { /* FFs doesn't get set all the time. */ }
8206 else
8207 return rcStrict;
8208
8209 /*
8210 * Setup the virtualized-APIC accesses.
8211 *
8212 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8213 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8214 *
8215 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8216 */
8217 PVM pVM = pVCpu->CTX_SUFF(pVM);
8218 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8219 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8220 && PDMHasApic(pVM))
8221 {
8222 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8223 Assert(u64MsrApicBase);
8224 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8225
8226 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8227
8228 /* Unalias any existing mapping. */
8229 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8230 AssertRCReturn(rc, rc);
8231
8232 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8233 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8234 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8235 AssertRCReturn(rc, rc);
8236
8237 /* Update the per-VCPU cache of the APIC base MSR. */
8238 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8239 }
8240
8241 if (TRPMHasTrap(pVCpu))
8242 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8243 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8244
8245 /*
8246 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8247 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8248 * also result in triple-faulting the VM.
8249 */
8250 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8251 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8252 { /* likely */ }
8253 else
8254 {
8255 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8256 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8257 return rcStrict;
8258 }
8259
8260 /*
8261 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8262 * import CR3 themselves. We will need to update them here, as even as late as the above
8263 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8264 * the below force flags to be set.
8265 */
8266 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8267 {
8268 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8269 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8270 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8271 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8272 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8273 }
8274 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8275 {
8276 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8277 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8278 }
8279
8280 /*
8281 * No longjmps to ring-3 from this point on!!!
8282 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8283 * This also disables flushing of the R0-logger instance (if any).
8284 */
8285 VMMRZCallRing3Disable(pVCpu);
8286
8287 /*
8288 * Export the guest state bits.
8289 *
8290 * We cannot perform longjmps while loading the guest state because we do not preserve the
8291 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8292 * CPU migration.
8293 *
8294 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8295 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8296 * Hence, loading of the guest state needs to be done -after- injection of events.
8297 */
8298 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8299 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8300 { /* likely */ }
8301 else
8302 {
8303 VMMRZCallRing3Enable(pVCpu);
8304 return rcStrict;
8305 }
8306
8307 /*
8308 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8309 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8310 * preemption disabled for a while. Since this is purly to aid the
8311 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8312 * disable interrupt on NT.
8313 *
8314 * We need to check for force-flags that could've possible been altered since we last
8315 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8316 * see @bugref{6398}).
8317 *
8318 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8319 * to ring-3 before executing guest code.
8320 */
8321 pVmxTransient->fEFlags = ASMIntDisableFlags();
8322
8323 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8324 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8325 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8326 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8327 {
8328 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8329 {
8330 pVCpu->hm.s.Event.fPending = false;
8331
8332 /*
8333 * We've injected any pending events. This is really the point of no return (to ring-3).
8334 *
8335 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8336 * returns from this function, so don't enable them here.
8337 */
8338 return VINF_SUCCESS;
8339 }
8340
8341 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8342 rcStrict = VINF_EM_RAW_INTERRUPT;
8343 }
8344 else
8345 {
8346 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8347 rcStrict = VINF_EM_RAW_TO_R3;
8348 }
8349
8350 ASMSetFlags(pVmxTransient->fEFlags);
8351 VMMRZCallRing3Enable(pVCpu);
8352
8353 return rcStrict;
8354}
8355
8356
8357/**
8358 * Prepares to run guest code in VT-x and we've committed to doing so. This
8359 * means there is no backing out to ring-3 or anywhere else at this
8360 * point.
8361 *
8362 * @param pVCpu The cross context virtual CPU structure.
8363 * @param pVmxTransient Pointer to the VMX transient structure.
8364 *
8365 * @remarks Called with preemption disabled.
8366 * @remarks No-long-jump zone!!!
8367 */
8368static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8369{
8370 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8371 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8372 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8373
8374 /*
8375 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8376 */
8377 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8378 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8379
8380 PVM pVM = pVCpu->CTX_SUFF(pVM);
8381 if (!CPUMIsGuestFPUStateActive(pVCpu))
8382 {
8383 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8384 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8385 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8386 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8387 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8388 }
8389
8390 /*
8391 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8392 */
8393 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8394 && pVCpu->hm.s.vmx.cMsrs > 0)
8395 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8396
8397 /*
8398 * Re-save the host state bits as we may've been preempted (only happens when
8399 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8400 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8401 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8402 * See @bugref{8432}.
8403 */
8404 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8405 {
8406 int rc = hmR0VmxExportHostState(pVCpu);
8407 AssertRC(rc);
8408 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8409 }
8410 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8411
8412 /*
8413 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8414 */
8415 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8416 hmR0VmxExportSharedState(pVCpu);
8417 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8418
8419 /* Store status of the shared guest-host state at the time of VM-entry. */
8420#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8421 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8422 {
8423 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8424 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8425 }
8426 else
8427#endif
8428 {
8429 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8430 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8431 }
8432
8433 /*
8434 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8435 */
8436 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8437 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8438
8439 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8440 RTCPUID idCurrentCpu = pCpu->idCpu;
8441 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8442 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8443 {
8444 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8445 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8446 }
8447
8448 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8449 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8450 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8451 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8452
8453 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8454
8455 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8456 to start executing. */
8457
8458 /*
8459 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8460 */
8461 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8462 {
8463 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8464 {
8465 bool fMsrUpdated;
8466 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8467 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8468 &fMsrUpdated);
8469 AssertRC(rc2);
8470 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8471 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8472 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8473 }
8474 else
8475 {
8476 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8477 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8478 }
8479 }
8480
8481 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8482 {
8483 bool fMsrUpdated;
8484 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8485 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8486 &fMsrUpdated);
8487 AssertRC(rc2);
8488 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8489 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8490 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8491 }
8492
8493#ifdef VBOX_STRICT
8494 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8495 hmR0VmxCheckHostEferMsr(pVCpu);
8496 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8497#endif
8498#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8499 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8500 {
8501 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8502 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8503 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8504 }
8505#endif
8506}
8507
8508
8509/**
8510 * Performs some essential restoration of state after running guest code in
8511 * VT-x.
8512 *
8513 * @param pVCpu The cross context virtual CPU structure.
8514 * @param pVmxTransient Pointer to the VMX transient structure.
8515 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8516 *
8517 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8518 *
8519 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8520 * unconditionally when it is safe to do so.
8521 */
8522static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8523{
8524 uint64_t const uHostTsc = ASMReadTSC();
8525 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8526
8527 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8528 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8529 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8530 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8531 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8532 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8533
8534 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8535 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8536
8537 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8538 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8539 Assert(!ASMIntAreEnabled());
8540 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8541
8542#if HC_ARCH_BITS == 64
8543 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8544#endif
8545#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8546 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8547 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8548 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8549#else
8550 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8551#endif
8552#ifdef VBOX_STRICT
8553 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8554#endif
8555 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8556
8557 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8558 uint32_t uExitReason;
8559 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8560 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8561 AssertRC(rc);
8562 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8563 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8564
8565 if (rcVMRun == VINF_SUCCESS)
8566 {
8567 /*
8568 * Update the VM-exit history array here even if the VM-entry failed due to:
8569 * - Invalid guest state.
8570 * - MSR loading.
8571 * - Machine-check event.
8572 *
8573 * In any of the above cases we will still have a "valid" VM-exit reason
8574 * despite @a fVMEntryFailed being false.
8575 *
8576 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8577 *
8578 * Note! We don't have CS or RIP at this point. Will probably address that later
8579 * by amending the history entry added here.
8580 */
8581 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8582 UINT64_MAX, uHostTsc);
8583
8584 if (!pVmxTransient->fVMEntryFailed)
8585 {
8586 VMMRZCallRing3Enable(pVCpu);
8587
8588 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8589 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8590
8591#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8592 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8593 AssertRC(rc);
8594#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8595 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8596 AssertRC(rc);
8597#else
8598 /*
8599 * Import the guest-interruptibility state always as we need it while evaluating
8600 * injecting events on re-entry.
8601 *
8602 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8603 * checking for real-mode while exporting the state because all bits that cause
8604 * mode changes wrt CR0 are intercepted.
8605 */
8606 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8607 AssertRC(rc);
8608#endif
8609
8610 /*
8611 * Sync the TPR shadow with our APIC state.
8612 */
8613 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8614 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8615 {
8616 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8617 AssertRC(rc);
8618 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8619 }
8620
8621 return;
8622 }
8623 }
8624 else
8625 {
8626 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8627 }
8628
8629 VMMRZCallRing3Enable(pVCpu);
8630}
8631
8632
8633/**
8634 * Runs the guest code using VT-x the normal way.
8635 *
8636 * @returns VBox status code.
8637 * @param pVCpu The cross context virtual CPU structure.
8638 *
8639 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8640 */
8641static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8642{
8643 VMXTRANSIENT VmxTransient;
8644 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8645 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8646 uint32_t cLoops = 0;
8647
8648 for (;; cLoops++)
8649 {
8650 Assert(!HMR0SuspendPending());
8651 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8652
8653 /* Preparatory work for running guest code, this may force us to return
8654 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8655 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8656 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8657 if (rcStrict != VINF_SUCCESS)
8658 break;
8659
8660 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8661 int rcRun = hmR0VmxRunGuest(pVCpu);
8662
8663 /* Restore any residual host-state and save any bits shared between host
8664 and guest into the guest-CPU state. Re-enables interrupts! */
8665 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8666
8667 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8668 if (RT_SUCCESS(rcRun))
8669 { /* very likely */ }
8670 else
8671 {
8672 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8673 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8674 return rcRun;
8675 }
8676
8677 /* Profile the VM-exit. */
8678 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8679 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8680 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8681 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8682 HMVMX_START_EXIT_DISPATCH_PROF();
8683
8684 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8685
8686 /* Handle the VM-exit. */
8687#ifdef HMVMX_USE_FUNCTION_TABLE
8688 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8689#else
8690 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8691#endif
8692 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8693 if (rcStrict == VINF_SUCCESS)
8694 {
8695 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8696 continue; /* likely */
8697 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8698 rcStrict = VINF_EM_RAW_INTERRUPT;
8699 }
8700 break;
8701 }
8702
8703 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8704 return rcStrict;
8705}
8706
8707
8708
8709/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8710 * probes.
8711 *
8712 * The following few functions and associated structure contains the bloat
8713 * necessary for providing detailed debug events and dtrace probes as well as
8714 * reliable host side single stepping. This works on the principle of
8715 * "subclassing" the normal execution loop and workers. We replace the loop
8716 * method completely and override selected helpers to add necessary adjustments
8717 * to their core operation.
8718 *
8719 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8720 * any performance for debug and analysis features.
8721 *
8722 * @{
8723 */
8724
8725/**
8726 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8727 * the debug run loop.
8728 */
8729typedef struct VMXRUNDBGSTATE
8730{
8731 /** The RIP we started executing at. This is for detecting that we stepped. */
8732 uint64_t uRipStart;
8733 /** The CS we started executing with. */
8734 uint16_t uCsStart;
8735
8736 /** Whether we've actually modified the 1st execution control field. */
8737 bool fModifiedProcCtls : 1;
8738 /** Whether we've actually modified the 2nd execution control field. */
8739 bool fModifiedProcCtls2 : 1;
8740 /** Whether we've actually modified the exception bitmap. */
8741 bool fModifiedXcptBitmap : 1;
8742
8743 /** We desire the modified the CR0 mask to be cleared. */
8744 bool fClearCr0Mask : 1;
8745 /** We desire the modified the CR4 mask to be cleared. */
8746 bool fClearCr4Mask : 1;
8747 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8748 uint32_t fCpe1Extra;
8749 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8750 uint32_t fCpe1Unwanted;
8751 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8752 uint32_t fCpe2Extra;
8753 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8754 uint32_t bmXcptExtra;
8755 /** The sequence number of the Dtrace provider settings the state was
8756 * configured against. */
8757 uint32_t uDtraceSettingsSeqNo;
8758 /** VM-exits to check (one bit per VM-exit). */
8759 uint32_t bmExitsToCheck[3];
8760
8761 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8762 uint32_t fProcCtlsInitial;
8763 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8764 uint32_t fProcCtls2Initial;
8765 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8766 uint32_t bmXcptInitial;
8767} VMXRUNDBGSTATE;
8768AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8769typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8770
8771
8772/**
8773 * Initializes the VMXRUNDBGSTATE structure.
8774 *
8775 * @param pVCpu The cross context virtual CPU structure of the
8776 * calling EMT.
8777 * @param pDbgState The structure to initialize.
8778 */
8779static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8780{
8781 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8782 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8783
8784 pDbgState->fModifiedProcCtls = false;
8785 pDbgState->fModifiedProcCtls2 = false;
8786 pDbgState->fModifiedXcptBitmap = false;
8787 pDbgState->fClearCr0Mask = false;
8788 pDbgState->fClearCr4Mask = false;
8789 pDbgState->fCpe1Extra = 0;
8790 pDbgState->fCpe1Unwanted = 0;
8791 pDbgState->fCpe2Extra = 0;
8792 pDbgState->bmXcptExtra = 0;
8793 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8794 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8795 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8796}
8797
8798
8799/**
8800 * Updates the VMSC fields with changes requested by @a pDbgState.
8801 *
8802 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8803 * immediately before executing guest code, i.e. when interrupts are disabled.
8804 * We don't check status codes here as we cannot easily assert or return in the
8805 * latter case.
8806 *
8807 * @param pVCpu The cross context virtual CPU structure.
8808 * @param pDbgState The debug state.
8809 */
8810static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8811{
8812 /*
8813 * Ensure desired flags in VMCS control fields are set.
8814 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8815 *
8816 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8817 * there should be no stale data in pCtx at this point.
8818 */
8819 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8820 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8821 {
8822 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8823 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8824 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8825 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8826 pDbgState->fModifiedProcCtls = true;
8827 }
8828
8829 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8830 {
8831 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8832 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8833 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8834 pDbgState->fModifiedProcCtls2 = true;
8835 }
8836
8837 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8838 {
8839 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8840 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8841 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8842 pDbgState->fModifiedXcptBitmap = true;
8843 }
8844
8845 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8846 {
8847 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8848 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8849 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8850 }
8851
8852 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8853 {
8854 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8855 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8856 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8857 }
8858}
8859
8860
8861static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8862{
8863 /*
8864 * Restore VM-exit control settings as we may not reenter this function the
8865 * next time around.
8866 */
8867 /* We reload the initial value, trigger what we can of recalculations the
8868 next time around. From the looks of things, that's all that's required atm. */
8869 if (pDbgState->fModifiedProcCtls)
8870 {
8871 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8872 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8873 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8874 AssertRCReturn(rc2, rc2);
8875 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8876 }
8877
8878 /* We're currently the only ones messing with this one, so just restore the
8879 cached value and reload the field. */
8880 if ( pDbgState->fModifiedProcCtls2
8881 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8882 {
8883 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8884 AssertRCReturn(rc2, rc2);
8885 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8886 }
8887
8888 /* If we've modified the exception bitmap, we restore it and trigger
8889 reloading and partial recalculation the next time around. */
8890 if (pDbgState->fModifiedXcptBitmap)
8891 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8892
8893 return rcStrict;
8894}
8895
8896
8897/**
8898 * Configures VM-exit controls for current DBGF and DTrace settings.
8899 *
8900 * This updates @a pDbgState and the VMCS execution control fields to reflect
8901 * the necessary VM-exits demanded by DBGF and DTrace.
8902 *
8903 * @param pVCpu The cross context virtual CPU structure.
8904 * @param pDbgState The debug state.
8905 * @param pVmxTransient Pointer to the VMX transient structure. May update
8906 * fUpdateTscOffsettingAndPreemptTimer.
8907 */
8908static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8909{
8910 /*
8911 * Take down the dtrace serial number so we can spot changes.
8912 */
8913 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8914 ASMCompilerBarrier();
8915
8916 /*
8917 * We'll rebuild most of the middle block of data members (holding the
8918 * current settings) as we go along here, so start by clearing it all.
8919 */
8920 pDbgState->bmXcptExtra = 0;
8921 pDbgState->fCpe1Extra = 0;
8922 pDbgState->fCpe1Unwanted = 0;
8923 pDbgState->fCpe2Extra = 0;
8924 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8925 pDbgState->bmExitsToCheck[i] = 0;
8926
8927 /*
8928 * Software interrupts (INT XXh) - no idea how to trigger these...
8929 */
8930 PVM pVM = pVCpu->CTX_SUFF(pVM);
8931 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8932 || VBOXVMM_INT_SOFTWARE_ENABLED())
8933 {
8934 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8935 }
8936
8937 /*
8938 * INT3 breakpoints - triggered by #BP exceptions.
8939 */
8940 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8941 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8942
8943 /*
8944 * Exception bitmap and XCPT events+probes.
8945 */
8946 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8947 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8948 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8949
8950 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8951 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8952 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8953 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8954 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8955 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8956 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8957 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8958 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8959 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8960 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8961 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8962 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8963 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8964 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8965 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8966 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8967 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8968
8969 if (pDbgState->bmXcptExtra)
8970 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8971
8972 /*
8973 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8974 *
8975 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8976 * So, when adding/changing/removing please don't forget to update it.
8977 *
8978 * Some of the macros are picking up local variables to save horizontal space,
8979 * (being able to see it in a table is the lesser evil here).
8980 */
8981#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8982 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8983 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8984#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8985 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8986 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8987 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8988 } else do { } while (0)
8989#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8990 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8991 { \
8992 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8993 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8994 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8995 } else do { } while (0)
8996#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8997 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8998 { \
8999 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9000 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9001 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9002 } else do { } while (0)
9003#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9004 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9005 { \
9006 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9007 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9008 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9009 } else do { } while (0)
9010
9011 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9012 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9013 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9014 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9015 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9016
9017 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9018 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9019 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9020 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9021 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9022 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9023 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9024 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9025 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9026 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9027 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9028 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9029 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9030 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9031 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9032 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9033 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9034 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9035 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9036 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9037 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9038 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9039 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9040 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9041 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9042 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9043 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9044 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9045 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9046 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9047 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9048 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9049 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9050 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9051 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9052 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9053
9054 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9055 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9056 {
9057 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9058 AssertRC(rc);
9059
9060#if 0 /** @todo fix me */
9061 pDbgState->fClearCr0Mask = true;
9062 pDbgState->fClearCr4Mask = true;
9063#endif
9064 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9065 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9066 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9067 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9068 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9069 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9070 require clearing here and in the loop if we start using it. */
9071 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9072 }
9073 else
9074 {
9075 if (pDbgState->fClearCr0Mask)
9076 {
9077 pDbgState->fClearCr0Mask = false;
9078 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9079 }
9080 if (pDbgState->fClearCr4Mask)
9081 {
9082 pDbgState->fClearCr4Mask = false;
9083 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9084 }
9085 }
9086 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9087 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9088
9089 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9090 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9091 {
9092 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9093 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9094 }
9095 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9096 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9097
9098 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9099 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9100 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9101 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9102 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9104 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9105 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9106#if 0 /** @todo too slow, fix handler. */
9107 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9108#endif
9109 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9110
9111 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9112 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9113 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9114 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9115 {
9116 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9117 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9118 }
9119 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9120 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9121 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9122 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9123
9124 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9125 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9126 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9127 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9128 {
9129 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9130 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9131 }
9132 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9133 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9134 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9135 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9136
9137 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9139 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9141 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9143 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9145 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9147 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9149 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9151 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9153 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9155 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9156 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9157 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9159
9160#undef IS_EITHER_ENABLED
9161#undef SET_ONLY_XBM_IF_EITHER_EN
9162#undef SET_CPE1_XBM_IF_EITHER_EN
9163#undef SET_CPEU_XBM_IF_EITHER_EN
9164#undef SET_CPE2_XBM_IF_EITHER_EN
9165
9166 /*
9167 * Sanitize the control stuff.
9168 */
9169 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9170 if (pDbgState->fCpe2Extra)
9171 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9172 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9173 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9174 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9175 {
9176 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9177 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9178 }
9179
9180 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9181 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9182 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9183 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9184}
9185
9186
9187/**
9188 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9189 * appropriate.
9190 *
9191 * The caller has checked the VM-exit against the
9192 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9193 * already, so we don't have to do that either.
9194 *
9195 * @returns Strict VBox status code (i.e. informational status codes too).
9196 * @param pVCpu The cross context virtual CPU structure.
9197 * @param pVmxTransient Pointer to the VMX-transient structure.
9198 * @param uExitReason The VM-exit reason.
9199 *
9200 * @remarks The name of this function is displayed by dtrace, so keep it short
9201 * and to the point. No longer than 33 chars long, please.
9202 */
9203static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9204{
9205 /*
9206 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9207 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9208 *
9209 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9210 * does. Must add/change/remove both places. Same ordering, please.
9211 *
9212 * Added/removed events must also be reflected in the next section
9213 * where we dispatch dtrace events.
9214 */
9215 bool fDtrace1 = false;
9216 bool fDtrace2 = false;
9217 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9218 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9219 uint32_t uEventArg = 0;
9220#define SET_EXIT(a_EventSubName) \
9221 do { \
9222 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9223 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9224 } while (0)
9225#define SET_BOTH(a_EventSubName) \
9226 do { \
9227 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9228 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9229 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9230 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9231 } while (0)
9232 switch (uExitReason)
9233 {
9234 case VMX_EXIT_MTF:
9235 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9236
9237 case VMX_EXIT_XCPT_OR_NMI:
9238 {
9239 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9240 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9241 {
9242 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9243 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9244 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9245 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9246 {
9247 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9248 {
9249 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9250 uEventArg = pVmxTransient->uExitIntErrorCode;
9251 }
9252 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9253 switch (enmEvent1)
9254 {
9255 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9256 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9257 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9258 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9259 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9260 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9261 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9262 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9263 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9264 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9265 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9266 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9267 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9268 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9269 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9270 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9271 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9272 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9273 default: break;
9274 }
9275 }
9276 else
9277 AssertFailed();
9278 break;
9279
9280 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9281 uEventArg = idxVector;
9282 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9283 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9284 break;
9285 }
9286 break;
9287 }
9288
9289 case VMX_EXIT_TRIPLE_FAULT:
9290 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9291 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9292 break;
9293 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9294 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9295 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9296 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9297 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9298
9299 /* Instruction specific VM-exits: */
9300 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9301 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9302 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9303 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9304 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9305 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9306 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9307 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9308 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9309 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9310 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9311 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9312 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9313 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9314 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9315 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9316 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9317 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9318 case VMX_EXIT_MOV_CRX:
9319 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9320 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9321 SET_BOTH(CRX_READ);
9322 else
9323 SET_BOTH(CRX_WRITE);
9324 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9325 break;
9326 case VMX_EXIT_MOV_DRX:
9327 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9328 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9329 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9330 SET_BOTH(DRX_READ);
9331 else
9332 SET_BOTH(DRX_WRITE);
9333 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9334 break;
9335 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9336 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9337 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9338 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9339 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9340 case VMX_EXIT_XDTR_ACCESS:
9341 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9342 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9343 {
9344 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9345 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9346 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9347 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9348 }
9349 break;
9350
9351 case VMX_EXIT_TR_ACCESS:
9352 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9353 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9354 {
9355 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9356 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9357 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9358 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9359 }
9360 break;
9361
9362 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9363 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9364 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9365 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9366 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9367 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9368 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9369 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9370 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9371 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9372 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9373
9374 /* Events that aren't relevant at this point. */
9375 case VMX_EXIT_EXT_INT:
9376 case VMX_EXIT_INT_WINDOW:
9377 case VMX_EXIT_NMI_WINDOW:
9378 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9379 case VMX_EXIT_PREEMPT_TIMER:
9380 case VMX_EXIT_IO_INSTR:
9381 break;
9382
9383 /* Errors and unexpected events. */
9384 case VMX_EXIT_INIT_SIGNAL:
9385 case VMX_EXIT_SIPI:
9386 case VMX_EXIT_IO_SMI:
9387 case VMX_EXIT_SMI:
9388 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9389 case VMX_EXIT_ERR_MSR_LOAD:
9390 case VMX_EXIT_ERR_MACHINE_CHECK:
9391 break;
9392
9393 default:
9394 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9395 break;
9396 }
9397#undef SET_BOTH
9398#undef SET_EXIT
9399
9400 /*
9401 * Dtrace tracepoints go first. We do them here at once so we don't
9402 * have to copy the guest state saving and stuff a few dozen times.
9403 * Down side is that we've got to repeat the switch, though this time
9404 * we use enmEvent since the probes are a subset of what DBGF does.
9405 */
9406 if (fDtrace1 || fDtrace2)
9407 {
9408 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9409 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9410 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9411 switch (enmEvent1)
9412 {
9413 /** @todo consider which extra parameters would be helpful for each probe. */
9414 case DBGFEVENT_END: break;
9415 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9416 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9417 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9418 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9419 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9420 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9421 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9422 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9424 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9425 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9426 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9427 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9428 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9429 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9430 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9431 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9432 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9433 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9434 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9435 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9436 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9437 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9438 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9439 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9440 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9443 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9444 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9445 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9446 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9447 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9448 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9449 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9450 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9451 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9452 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9453 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9454 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9481 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9482 }
9483 switch (enmEvent2)
9484 {
9485 /** @todo consider which extra parameters would be helpful for each probe. */
9486 case DBGFEVENT_END: break;
9487 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9488 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9489 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9490 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9491 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9492 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9493 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9494 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9497 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9498 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9499 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9500 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9501 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9502 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9503 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9504 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9505 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9506 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9507 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9508 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9539 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9540 }
9541 }
9542
9543 /*
9544 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9545 * the DBGF call will do a full check).
9546 *
9547 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9548 * Note! If we have to events, we prioritize the first, i.e. the instruction
9549 * one, in order to avoid event nesting.
9550 */
9551 PVM pVM = pVCpu->CTX_SUFF(pVM);
9552 if ( enmEvent1 != DBGFEVENT_END
9553 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9554 {
9555 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9556 if (rcStrict != VINF_SUCCESS)
9557 return rcStrict;
9558 }
9559 else if ( enmEvent2 != DBGFEVENT_END
9560 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9561 {
9562 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9563 if (rcStrict != VINF_SUCCESS)
9564 return rcStrict;
9565 }
9566
9567 return VINF_SUCCESS;
9568}
9569
9570
9571/**
9572 * Single-stepping VM-exit filtering.
9573 *
9574 * This is preprocessing the VM-exits and deciding whether we've gotten far
9575 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9576 * handling is performed.
9577 *
9578 * @returns Strict VBox status code (i.e. informational status codes too).
9579 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9580 * @param pVmxTransient Pointer to the VMX-transient structure.
9581 * @param pDbgState The debug state.
9582 */
9583DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9584{
9585 /*
9586 * Expensive (saves context) generic dtrace VM-exit probe.
9587 */
9588 uint32_t const uExitReason = pVmxTransient->uExitReason;
9589 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9590 { /* more likely */ }
9591 else
9592 {
9593 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9594 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9595 AssertRC(rc);
9596 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9597 }
9598
9599 /*
9600 * Check for host NMI, just to get that out of the way.
9601 */
9602 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9603 { /* normally likely */ }
9604 else
9605 {
9606 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9607 AssertRCReturn(rc2, rc2);
9608 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9609 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9610 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9611 }
9612
9613 /*
9614 * Check for single stepping event if we're stepping.
9615 */
9616 if (pVCpu->hm.s.fSingleInstruction)
9617 {
9618 switch (uExitReason)
9619 {
9620 case VMX_EXIT_MTF:
9621 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9622
9623 /* Various events: */
9624 case VMX_EXIT_XCPT_OR_NMI:
9625 case VMX_EXIT_EXT_INT:
9626 case VMX_EXIT_TRIPLE_FAULT:
9627 case VMX_EXIT_INT_WINDOW:
9628 case VMX_EXIT_NMI_WINDOW:
9629 case VMX_EXIT_TASK_SWITCH:
9630 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9631 case VMX_EXIT_APIC_ACCESS:
9632 case VMX_EXIT_EPT_VIOLATION:
9633 case VMX_EXIT_EPT_MISCONFIG:
9634 case VMX_EXIT_PREEMPT_TIMER:
9635
9636 /* Instruction specific VM-exits: */
9637 case VMX_EXIT_CPUID:
9638 case VMX_EXIT_GETSEC:
9639 case VMX_EXIT_HLT:
9640 case VMX_EXIT_INVD:
9641 case VMX_EXIT_INVLPG:
9642 case VMX_EXIT_RDPMC:
9643 case VMX_EXIT_RDTSC:
9644 case VMX_EXIT_RSM:
9645 case VMX_EXIT_VMCALL:
9646 case VMX_EXIT_VMCLEAR:
9647 case VMX_EXIT_VMLAUNCH:
9648 case VMX_EXIT_VMPTRLD:
9649 case VMX_EXIT_VMPTRST:
9650 case VMX_EXIT_VMREAD:
9651 case VMX_EXIT_VMRESUME:
9652 case VMX_EXIT_VMWRITE:
9653 case VMX_EXIT_VMXOFF:
9654 case VMX_EXIT_VMXON:
9655 case VMX_EXIT_MOV_CRX:
9656 case VMX_EXIT_MOV_DRX:
9657 case VMX_EXIT_IO_INSTR:
9658 case VMX_EXIT_RDMSR:
9659 case VMX_EXIT_WRMSR:
9660 case VMX_EXIT_MWAIT:
9661 case VMX_EXIT_MONITOR:
9662 case VMX_EXIT_PAUSE:
9663 case VMX_EXIT_XDTR_ACCESS:
9664 case VMX_EXIT_TR_ACCESS:
9665 case VMX_EXIT_INVEPT:
9666 case VMX_EXIT_RDTSCP:
9667 case VMX_EXIT_INVVPID:
9668 case VMX_EXIT_WBINVD:
9669 case VMX_EXIT_XSETBV:
9670 case VMX_EXIT_RDRAND:
9671 case VMX_EXIT_INVPCID:
9672 case VMX_EXIT_VMFUNC:
9673 case VMX_EXIT_RDSEED:
9674 case VMX_EXIT_XSAVES:
9675 case VMX_EXIT_XRSTORS:
9676 {
9677 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9678 AssertRCReturn(rc, rc);
9679 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9680 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9681 return VINF_EM_DBG_STEPPED;
9682 break;
9683 }
9684
9685 /* Errors and unexpected events: */
9686 case VMX_EXIT_INIT_SIGNAL:
9687 case VMX_EXIT_SIPI:
9688 case VMX_EXIT_IO_SMI:
9689 case VMX_EXIT_SMI:
9690 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9691 case VMX_EXIT_ERR_MSR_LOAD:
9692 case VMX_EXIT_ERR_MACHINE_CHECK:
9693 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9694 break;
9695
9696 default:
9697 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9698 break;
9699 }
9700 }
9701
9702 /*
9703 * Check for debugger event breakpoints and dtrace probes.
9704 */
9705 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9706 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9707 {
9708 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9709 if (rcStrict != VINF_SUCCESS)
9710 return rcStrict;
9711 }
9712
9713 /*
9714 * Normal processing.
9715 */
9716#ifdef HMVMX_USE_FUNCTION_TABLE
9717 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9718#else
9719 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9720#endif
9721}
9722
9723
9724/**
9725 * Single steps guest code using VT-x.
9726 *
9727 * @returns Strict VBox status code (i.e. informational status codes too).
9728 * @param pVCpu The cross context virtual CPU structure.
9729 *
9730 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9731 */
9732static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9733{
9734 VMXTRANSIENT VmxTransient;
9735 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9736
9737 /* Set HMCPU indicators. */
9738 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9739 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9740 pVCpu->hm.s.fDebugWantRdTscExit = false;
9741 pVCpu->hm.s.fUsingDebugLoop = true;
9742
9743 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9744 VMXRUNDBGSTATE DbgState;
9745 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9746 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9747
9748 /*
9749 * The loop.
9750 */
9751 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9752 for (uint32_t cLoops = 0; ; cLoops++)
9753 {
9754 Assert(!HMR0SuspendPending());
9755 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9756 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9757
9758 /*
9759 * Preparatory work for running guest code, this may force us to return
9760 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9761 */
9762 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9763 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9764 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9765 if (rcStrict != VINF_SUCCESS)
9766 break;
9767
9768 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9769 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9770
9771 /*
9772 * Now we can run the guest code.
9773 */
9774 int rcRun = hmR0VmxRunGuest(pVCpu);
9775
9776 /*
9777 * Restore any residual host-state and save any bits shared between host
9778 * and guest into the guest-CPU state. Re-enables interrupts!
9779 */
9780 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9781
9782 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9783 if (RT_SUCCESS(rcRun))
9784 { /* very likely */ }
9785 else
9786 {
9787 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9788 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9789 return rcRun;
9790 }
9791
9792 /* Profile the VM-exit. */
9793 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9794 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9795 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9796 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9797 HMVMX_START_EXIT_DISPATCH_PROF();
9798
9799 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9800
9801 /*
9802 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9803 */
9804 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9805 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9806 if (rcStrict != VINF_SUCCESS)
9807 break;
9808 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9809 {
9810 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9811 rcStrict = VINF_EM_RAW_INTERRUPT;
9812 break;
9813 }
9814
9815 /*
9816 * Stepping: Did the RIP change, if so, consider it a single step.
9817 * Otherwise, make sure one of the TFs gets set.
9818 */
9819 if (fStepping)
9820 {
9821 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9822 AssertRC(rc);
9823 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9824 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9825 {
9826 rcStrict = VINF_EM_DBG_STEPPED;
9827 break;
9828 }
9829 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9830 }
9831
9832 /*
9833 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9834 */
9835 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9836 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9837 }
9838
9839 /*
9840 * Clear the X86_EFL_TF if necessary.
9841 */
9842 if (pVCpu->hm.s.fClearTrapFlag)
9843 {
9844 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9845 AssertRC(rc);
9846 pVCpu->hm.s.fClearTrapFlag = false;
9847 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9848 }
9849 /** @todo there seems to be issues with the resume flag when the monitor trap
9850 * flag is pending without being used. Seen early in bios init when
9851 * accessing APIC page in protected mode. */
9852
9853 /*
9854 * Restore VM-exit control settings as we may not reenter this function the
9855 * next time around.
9856 */
9857 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9858
9859 /* Restore HMCPU indicators. */
9860 pVCpu->hm.s.fUsingDebugLoop = false;
9861 pVCpu->hm.s.fDebugWantRdTscExit = false;
9862 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9863
9864 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9865 return rcStrict;
9866}
9867
9868
9869/** @} */
9870
9871
9872/**
9873 * Checks if any expensive dtrace probes are enabled and we should go to the
9874 * debug loop.
9875 *
9876 * @returns true if we should use debug loop, false if not.
9877 */
9878static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9879{
9880 /* It's probably faster to OR the raw 32-bit counter variables together.
9881 Since the variables are in an array and the probes are next to one
9882 another (more or less), we have good locality. So, better read
9883 eight-nine cache lines ever time and only have one conditional, than
9884 128+ conditionals, right? */
9885 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9886 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9887 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9888 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9889 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9890 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9891 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9892 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9893 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9894 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9895 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9896 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9897 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9898 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9899 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9900 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9901 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9902 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9903 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9904 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9905 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9906 ) != 0
9907 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9908 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9909 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9910 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9911 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9912 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9913 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9914 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9915 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9916 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9917 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9918 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9919 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9920 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9921 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9922 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9923 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9924 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9925 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9926 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9927 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9928 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9929 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9930 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9932 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9933 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9934 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9935 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9936 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9937 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9938 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9939 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9940 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9941 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9942 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9943 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9953 ) != 0
9954 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9955 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9956 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9957 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9958 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9959 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9960 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9961 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9962 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9963 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9964 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9965 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9966 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9967 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9968 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9969 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9970 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9971 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9972 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9973 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9974 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9975 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9976 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9977 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9978 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9980 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9981 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9982 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9983 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9984 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9985 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9986 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9987 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9988 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9989 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9990 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9991 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10005 ) != 0;
10006}
10007
10008
10009/**
10010 * Runs the guest code using VT-x.
10011 *
10012 * @returns Strict VBox status code (i.e. informational status codes too).
10013 * @param pVCpu The cross context virtual CPU structure.
10014 */
10015VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10016{
10017 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10018 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10019 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10020 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10021
10022 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10023
10024 VBOXSTRICTRC rcStrict;
10025 if ( !pVCpu->hm.s.fUseDebugLoop
10026 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10027 && !DBGFIsStepping(pVCpu)
10028 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10029 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10030 else
10031 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10032
10033 if (rcStrict == VERR_EM_INTERPRETER)
10034 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10035 else if (rcStrict == VINF_EM_RESET)
10036 rcStrict = VINF_EM_TRIPLE_FAULT;
10037
10038 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10039 if (RT_FAILURE(rc2))
10040 {
10041 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10042 rcStrict = rc2;
10043 }
10044 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10045 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10046 return rcStrict;
10047}
10048
10049
10050#ifndef HMVMX_USE_FUNCTION_TABLE
10051DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10052{
10053#ifdef DEBUG_ramshankar
10054#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10055 do { \
10056 if (a_fSave != 0) \
10057 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10058 VBOXSTRICTRC rcStrict = a_CallExpr; \
10059 if (a_fSave != 0) \
10060 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10061 return rcStrict; \
10062 } while (0)
10063#else
10064# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10065#endif
10066 switch (rcReason)
10067 {
10068 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10069 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10070 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10071 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10072 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10073 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10074 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10075 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10076 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10077 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10078 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10079 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10080 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10081 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10082 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10083 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10084 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10085 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10086 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10087 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10088 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10089 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10090 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10091 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10092 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10093 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10094 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10095 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10096 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10097 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10098 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10099 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10100 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10101 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10102
10103 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10104 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10105 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10106 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10107 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10108 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10109 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10110 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10111 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10112
10113 case VMX_EXIT_VMCLEAR:
10114 case VMX_EXIT_VMLAUNCH:
10115 case VMX_EXIT_VMPTRLD:
10116 case VMX_EXIT_VMPTRST:
10117 case VMX_EXIT_VMREAD:
10118 case VMX_EXIT_VMRESUME:
10119 case VMX_EXIT_VMWRITE:
10120 case VMX_EXIT_VMXOFF:
10121 case VMX_EXIT_VMXON:
10122 case VMX_EXIT_INVEPT:
10123 case VMX_EXIT_INVVPID:
10124 case VMX_EXIT_VMFUNC:
10125 case VMX_EXIT_XSAVES:
10126 case VMX_EXIT_XRSTORS:
10127 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10128
10129 case VMX_EXIT_ENCLS:
10130 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10131 case VMX_EXIT_PML_FULL:
10132 default:
10133 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10134 }
10135#undef VMEXIT_CALL_RET
10136}
10137#endif /* !HMVMX_USE_FUNCTION_TABLE */
10138
10139
10140#ifdef VBOX_STRICT
10141/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10142# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10143 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10144
10145# define HMVMX_ASSERT_PREEMPT_CPUID() \
10146 do { \
10147 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10148 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10149 } while (0)
10150
10151# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10152 do { \
10153 AssertPtr((a_pVCpu)); \
10154 AssertPtr((a_pVmxTransient)); \
10155 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10156 Assert(ASMIntAreEnabled()); \
10157 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10158 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10159 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10160 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10161 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10162 HMVMX_ASSERT_PREEMPT_CPUID(); \
10163 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10164 } while (0)
10165
10166# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10167 do { \
10168 Log4Func(("\n")); \
10169 } while (0)
10170#else
10171# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10172 do { \
10173 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10174 RT_NOREF2((a_pVCpu), (a_pVmxTransient)); \
10175 } while (0)
10176# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10177#endif
10178
10179
10180/**
10181 * Advances the guest RIP by the specified number of bytes.
10182 *
10183 * @param pVCpu The cross context virtual CPU structure.
10184 * @param cbInstr Number of bytes to advance the RIP by.
10185 *
10186 * @remarks No-long-jump zone!!!
10187 */
10188DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10189{
10190 /* Advance the RIP. */
10191 pVCpu->cpum.GstCtx.rip += cbInstr;
10192 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10193
10194 /* Update interrupt inhibition. */
10195 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10196 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10197 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10198}
10199
10200
10201/**
10202 * Advances the guest RIP after reading it from the VMCS.
10203 *
10204 * @returns VBox status code, no informational status codes.
10205 * @param pVCpu The cross context virtual CPU structure.
10206 * @param pVmxTransient Pointer to the VMX transient structure.
10207 *
10208 * @remarks No-long-jump zone!!!
10209 */
10210static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10211{
10212 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10213 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10214 AssertRCReturn(rc, rc);
10215
10216 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10217
10218 /*
10219 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10220 * pending debug exception field as it takes care of priority of events.
10221 *
10222 * See Intel spec. 32.2.1 "Debug Exceptions".
10223 */
10224 if ( !pVCpu->hm.s.fSingleInstruction
10225 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10226 {
10227 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10228 AssertRCReturn(rc, rc);
10229 }
10230
10231 return VINF_SUCCESS;
10232}
10233
10234
10235/**
10236 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10237 * and update error record fields accordingly.
10238 *
10239 * @return VMX_IGS_* return codes.
10240 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10241 * wrong with the guest state.
10242 *
10243 * @param pVCpu The cross context virtual CPU structure.
10244 *
10245 * @remarks This function assumes our cache of the VMCS controls
10246 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10247 */
10248static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10249{
10250#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10251#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10252 uError = (err); \
10253 break; \
10254 } else do { } while (0)
10255
10256 int rc;
10257 PVM pVM = pVCpu->CTX_SUFF(pVM);
10258 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10259 uint32_t uError = VMX_IGS_ERROR;
10260 uint32_t u32Val;
10261 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10262
10263 do
10264 {
10265 /*
10266 * CR0.
10267 */
10268 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10269 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10270 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10271 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10272 if (fUnrestrictedGuest)
10273 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10274
10275 uint32_t u32GuestCr0;
10276 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10277 AssertRCBreak(rc);
10278 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10279 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10280 if ( !fUnrestrictedGuest
10281 && (u32GuestCr0 & X86_CR0_PG)
10282 && !(u32GuestCr0 & X86_CR0_PE))
10283 {
10284 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10285 }
10286
10287 /*
10288 * CR4.
10289 */
10290 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10291 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10292
10293 uint32_t u32GuestCr4;
10294 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10295 AssertRCBreak(rc);
10296 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10297 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10298
10299 /*
10300 * IA32_DEBUGCTL MSR.
10301 */
10302 uint64_t u64Val;
10303 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10304 AssertRCBreak(rc);
10305 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10306 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10307 {
10308 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10309 }
10310 uint64_t u64DebugCtlMsr = u64Val;
10311
10312#ifdef VBOX_STRICT
10313 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10314 AssertRCBreak(rc);
10315 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10316#endif
10317 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10318
10319 /*
10320 * RIP and RFLAGS.
10321 */
10322 uint32_t u32Eflags;
10323#if HC_ARCH_BITS == 64
10324 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10325 AssertRCBreak(rc);
10326 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10327 if ( !fLongModeGuest
10328 || !pCtx->cs.Attr.n.u1Long)
10329 {
10330 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10331 }
10332 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10333 * must be identical if the "IA-32e mode guest" VM-entry
10334 * control is 1 and CS.L is 1. No check applies if the
10335 * CPU supports 64 linear-address bits. */
10336
10337 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10338 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10339 AssertRCBreak(rc);
10340 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10341 VMX_IGS_RFLAGS_RESERVED);
10342 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10343 u32Eflags = u64Val;
10344#else
10345 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10346 AssertRCBreak(rc);
10347 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10348 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10349#endif
10350
10351 if ( fLongModeGuest
10352 || ( fUnrestrictedGuest
10353 && !(u32GuestCr0 & X86_CR0_PE)))
10354 {
10355 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10356 }
10357
10358 uint32_t u32EntryInfo;
10359 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10360 AssertRCBreak(rc);
10361 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10362 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10363 {
10364 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10365 }
10366
10367 /*
10368 * 64-bit checks.
10369 */
10370#if HC_ARCH_BITS == 64
10371 if (fLongModeGuest)
10372 {
10373 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10374 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10375 }
10376
10377 if ( !fLongModeGuest
10378 && (u32GuestCr4 & X86_CR4_PCIDE))
10379 {
10380 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10381 }
10382
10383 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10384 * 51:32 beyond the processor's physical-address width are 0. */
10385
10386 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10387 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10388 {
10389 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10390 }
10391
10392 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10393 AssertRCBreak(rc);
10394 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10395
10396 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10397 AssertRCBreak(rc);
10398 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10399#endif
10400
10401 /*
10402 * PERF_GLOBAL MSR.
10403 */
10404 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10405 {
10406 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10407 AssertRCBreak(rc);
10408 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10409 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10410 }
10411
10412 /*
10413 * PAT MSR.
10414 */
10415 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10416 {
10417 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10418 AssertRCBreak(rc);
10419 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10420 for (unsigned i = 0; i < 8; i++)
10421 {
10422 uint8_t u8Val = (u64Val & 0xff);
10423 if ( u8Val != 0 /* UC */
10424 && u8Val != 1 /* WC */
10425 && u8Val != 4 /* WT */
10426 && u8Val != 5 /* WP */
10427 && u8Val != 6 /* WB */
10428 && u8Val != 7 /* UC- */)
10429 {
10430 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10431 }
10432 u64Val >>= 8;
10433 }
10434 }
10435
10436 /*
10437 * EFER MSR.
10438 */
10439 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10440 {
10441 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10442 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10443 AssertRCBreak(rc);
10444 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10445 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10446 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10447 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10448 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10449 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10450 || !(u32GuestCr0 & X86_CR0_PG)
10451 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10452 VMX_IGS_EFER_LMA_LME_MISMATCH);
10453 }
10454
10455 /*
10456 * Segment registers.
10457 */
10458 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10459 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10460 if (!(u32Eflags & X86_EFL_VM))
10461 {
10462 /* CS */
10463 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10464 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10465 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10466 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10467 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10468 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10469 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10470 /* CS cannot be loaded with NULL in protected mode. */
10471 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10472 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10473 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10474 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10475 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10476 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10477 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10479 else
10480 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10481
10482 /* SS */
10483 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10484 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10485 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10486 if ( !(pCtx->cr0 & X86_CR0_PE)
10487 || pCtx->cs.Attr.n.u4Type == 3)
10488 {
10489 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10490 }
10491 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10492 {
10493 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10494 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10495 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10496 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10497 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10498 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10499 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10500 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10501 }
10502
10503 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10504 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10505 {
10506 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10507 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10508 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10509 || pCtx->ds.Attr.n.u4Type > 11
10510 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10511 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10512 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10513 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10514 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10515 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10516 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10517 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10518 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10519 }
10520 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10521 {
10522 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10523 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10524 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10525 || pCtx->es.Attr.n.u4Type > 11
10526 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10527 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10528 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10529 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10530 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10531 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10532 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10533 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10534 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10535 }
10536 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10537 {
10538 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10539 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10540 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10541 || pCtx->fs.Attr.n.u4Type > 11
10542 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10543 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10544 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10545 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10546 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10547 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10548 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10549 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10550 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10551 }
10552 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10553 {
10554 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10555 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10556 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10557 || pCtx->gs.Attr.n.u4Type > 11
10558 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10559 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10560 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10561 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10562 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10563 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10564 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10565 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10566 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10567 }
10568 /* 64-bit capable CPUs. */
10569#if HC_ARCH_BITS == 64
10570 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10571 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10572 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10573 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10574 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10575 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10576 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10577 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10578 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10579 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10580 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10581#endif
10582 }
10583 else
10584 {
10585 /* V86 mode checks. */
10586 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10587 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10588 {
10589 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10590 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10591 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10592 }
10593 else
10594 {
10595 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10596 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10597 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10598 }
10599
10600 /* CS */
10601 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10602 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10603 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10604 /* SS */
10605 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10606 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10607 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10608 /* DS */
10609 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10610 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10611 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10612 /* ES */
10613 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10614 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10615 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10616 /* FS */
10617 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10618 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10619 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10620 /* GS */
10621 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10622 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10623 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10624 /* 64-bit capable CPUs. */
10625#if HC_ARCH_BITS == 64
10626 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10627 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10628 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10629 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10630 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10631 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10632 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10633 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10634 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10635 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10636 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10637#endif
10638 }
10639
10640 /*
10641 * TR.
10642 */
10643 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10644 /* 64-bit capable CPUs. */
10645#if HC_ARCH_BITS == 64
10646 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10647#endif
10648 if (fLongModeGuest)
10649 {
10650 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10651 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10652 }
10653 else
10654 {
10655 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10656 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10657 VMX_IGS_TR_ATTR_TYPE_INVALID);
10658 }
10659 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10660 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10661 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10662 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10663 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10664 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10665 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10666 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10667
10668 /*
10669 * GDTR and IDTR.
10670 */
10671#if HC_ARCH_BITS == 64
10672 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10673 AssertRCBreak(rc);
10674 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10675
10676 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10677 AssertRCBreak(rc);
10678 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10679#endif
10680
10681 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10682 AssertRCBreak(rc);
10683 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10684
10685 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10686 AssertRCBreak(rc);
10687 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10688
10689 /*
10690 * Guest Non-Register State.
10691 */
10692 /* Activity State. */
10693 uint32_t u32ActivityState;
10694 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10695 AssertRCBreak(rc);
10696 HMVMX_CHECK_BREAK( !u32ActivityState
10697 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10698 VMX_IGS_ACTIVITY_STATE_INVALID);
10699 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10700 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10701 uint32_t u32IntrState;
10702 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10703 AssertRCBreak(rc);
10704 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10705 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10706 {
10707 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10708 }
10709
10710 /** @todo Activity state and injecting interrupts. Left as a todo since we
10711 * currently don't use activity states but ACTIVE. */
10712
10713 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10714 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10715
10716 /* Guest interruptibility-state. */
10717 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10718 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10719 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10720 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10721 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10722 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10723 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10724 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10725 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10726 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10727 {
10728 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10729 {
10730 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10731 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10732 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10733 }
10734 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10735 {
10736 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10737 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10738 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10739 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10740 }
10741 }
10742 /** @todo Assumes the processor is not in SMM. */
10743 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10744 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10745 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10746 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10747 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10748 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10749 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10750 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10751 {
10752 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10753 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10754 }
10755
10756 /* Pending debug exceptions. */
10757#if HC_ARCH_BITS == 64
10758 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10759 AssertRCBreak(rc);
10760 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10761 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10762 u32Val = u64Val; /* For pending debug exceptions checks below. */
10763#else
10764 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10765 AssertRCBreak(rc);
10766 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10767 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10768#endif
10769
10770 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10771 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10772 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10773 {
10774 if ( (u32Eflags & X86_EFL_TF)
10775 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10776 {
10777 /* Bit 14 is PendingDebug.BS. */
10778 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10779 }
10780 if ( !(u32Eflags & X86_EFL_TF)
10781 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10782 {
10783 /* Bit 14 is PendingDebug.BS. */
10784 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10785 }
10786 }
10787
10788 /* VMCS link pointer. */
10789 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10790 AssertRCBreak(rc);
10791 if (u64Val != UINT64_C(0xffffffffffffffff))
10792 {
10793 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10794 /** @todo Bits beyond the processor's physical-address width MBZ. */
10795 /** @todo 32-bit located in memory referenced by value of this field (as a
10796 * physical address) must contain the processor's VMCS revision ID. */
10797 /** @todo SMM checks. */
10798 }
10799
10800 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10801 * not using Nested Paging? */
10802 if ( pVM->hm.s.fNestedPaging
10803 && !fLongModeGuest
10804 && CPUMIsGuestInPAEModeEx(pCtx))
10805 {
10806 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10807 AssertRCBreak(rc);
10808 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10809
10810 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10811 AssertRCBreak(rc);
10812 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10813
10814 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10815 AssertRCBreak(rc);
10816 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10817
10818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10821 }
10822
10823 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10824 if (uError == VMX_IGS_ERROR)
10825 uError = VMX_IGS_REASON_NOT_FOUND;
10826 } while (0);
10827
10828 pVCpu->hm.s.u32HMError = uError;
10829 return uError;
10830
10831#undef HMVMX_ERROR_BREAK
10832#undef HMVMX_CHECK_BREAK
10833}
10834
10835/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10836/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10837/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10838
10839/** @name VM-exit handlers.
10840 * @{
10841 */
10842
10843/**
10844 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10845 */
10846HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10847{
10848 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10850 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10851 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10852 return VINF_SUCCESS;
10853 return VINF_EM_RAW_INTERRUPT;
10854}
10855
10856
10857/**
10858 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10859 */
10860HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10861{
10862 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10863 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10864
10865 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10866 AssertRCReturn(rc, rc);
10867
10868 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10869 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10870 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10871 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10872
10873 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10874 {
10875 /*
10876 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10877 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10878 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10879 *
10880 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10881 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10882 */
10883 VMXDispatchHostNmi();
10884 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10885 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10886 return VINF_SUCCESS;
10887 }
10888
10889 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10890 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10891 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10892 { /* likely */ }
10893 else
10894 {
10895 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10896 rcStrictRc1 = VINF_SUCCESS;
10897 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10898 return rcStrictRc1;
10899 }
10900
10901 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10902 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10903 switch (uIntType)
10904 {
10905 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10906 Assert(uVector == X86_XCPT_DB);
10907 RT_FALL_THRU();
10908 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10909 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10910 RT_FALL_THRU();
10911 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10912 {
10913 /*
10914 * If there's any exception caused as a result of event injection, the resulting
10915 * secondary/final execption will be pending, we shall continue guest execution
10916 * after injecting the event. The page-fault case is complicated and we manually
10917 * handle any currently pending event in hmR0VmxExitXcptPF.
10918 */
10919 if (!pVCpu->hm.s.Event.fPending)
10920 { /* likely */ }
10921 else if (uVector != X86_XCPT_PF)
10922 {
10923 rc = VINF_SUCCESS;
10924 break;
10925 }
10926
10927 switch (uVector)
10928 {
10929 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10930 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10931 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10932 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10933 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10934 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10935
10936 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10937 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10938 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10939 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10940 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10941 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10942 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10943 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10944 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10945 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10946 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10947 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10948 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10949 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10950 default:
10951 {
10952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10954 {
10955 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10956 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10957 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10958
10959 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10960 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10961 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10962 AssertRCReturn(rc, rc);
10963 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10964 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10965 0 /* GCPtrFaultAddress */);
10966 }
10967 else
10968 {
10969 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10970 pVCpu->hm.s.u32HMError = uVector;
10971 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10972 }
10973 break;
10974 }
10975 }
10976 break;
10977 }
10978
10979 default:
10980 {
10981 pVCpu->hm.s.u32HMError = uExitIntInfo;
10982 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10983 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10984 break;
10985 }
10986 }
10987 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10988 return rc;
10989}
10990
10991
10992/**
10993 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10994 */
10995HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10996{
10997 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10998
10999 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11000 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11001
11002 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11004 return VINF_SUCCESS;
11005}
11006
11007
11008/**
11009 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11010 */
11011HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11012{
11013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11014 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11015 {
11016 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11017 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11018 }
11019
11020 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11021
11022 /*
11023 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11024 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11025 */
11026 uint32_t fIntrState = 0;
11027 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11028 AssertRCReturn(rc, rc);
11029
11030 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11031 if ( fBlockSti
11032 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11033 {
11034 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11035 }
11036
11037 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11038 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11039
11040 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11041 return VINF_SUCCESS;
11042}
11043
11044
11045/**
11046 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11047 */
11048HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11049{
11050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11051 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11052}
11053
11054
11055/**
11056 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11057 */
11058HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11059{
11060 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11061 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11062}
11063
11064
11065/**
11066 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11067 */
11068HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11069{
11070 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11071
11072 /*
11073 * Get the state we need and update the exit history entry.
11074 */
11075 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11076 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11077 AssertRCReturn(rc, rc);
11078
11079 VBOXSTRICTRC rcStrict;
11080 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11081 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11082 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11083 if (!pExitRec)
11084 {
11085 /*
11086 * Regular CPUID instruction execution.
11087 */
11088 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11089 if (rcStrict == VINF_SUCCESS)
11090 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11091 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11092 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11093 {
11094 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11095 rcStrict = VINF_SUCCESS;
11096 }
11097 }
11098 else
11099 {
11100 /*
11101 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11102 */
11103 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11104 AssertRCReturn(rc2, rc2);
11105
11106 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11107 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11108
11109 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11110 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11111
11112 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11113 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11114 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11115 }
11116 return rcStrict;
11117}
11118
11119
11120/**
11121 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11122 */
11123HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11126 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11127 AssertRCReturn(rc, rc);
11128
11129 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11130 return VINF_EM_RAW_EMULATE_INSTR;
11131
11132 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11133 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11134}
11135
11136
11137/**
11138 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11139 */
11140HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11141{
11142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11143 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11144 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11145 AssertRCReturn(rc, rc);
11146
11147 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11148 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11149 {
11150 /* If we get a spurious VM-exit when offsetting is enabled,
11151 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11152 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11153 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11154 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11155 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11156 }
11157 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11158 {
11159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11160 rcStrict = VINF_SUCCESS;
11161 }
11162 return rcStrict;
11163}
11164
11165
11166/**
11167 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11168 */
11169HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11170{
11171 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11172 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11173 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11174 AssertRCReturn(rc, rc);
11175
11176 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11177 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11178 {
11179 /* If we get a spurious VM-exit when offsetting is enabled,
11180 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11181 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11182 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11183 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11184 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11185 }
11186 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11187 {
11188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11189 rcStrict = VINF_SUCCESS;
11190 }
11191 return rcStrict;
11192}
11193
11194
11195/**
11196 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11197 */
11198HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11199{
11200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11201 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11202 AssertRCReturn(rc, rc);
11203
11204 PVM pVM = pVCpu->CTX_SUFF(pVM);
11205 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11206 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11207 if (RT_LIKELY(rc == VINF_SUCCESS))
11208 {
11209 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11210 Assert(pVmxTransient->cbInstr == 2);
11211 }
11212 else
11213 {
11214 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11215 rc = VERR_EM_INTERPRETER;
11216 }
11217 return rc;
11218}
11219
11220
11221/**
11222 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11223 */
11224HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11225{
11226 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11227
11228 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11229 if (EMAreHypercallInstructionsEnabled(pVCpu))
11230 {
11231 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11232 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11233 AssertRCReturn(rc, rc);
11234
11235 /* Perform the hypercall. */
11236 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11237 if (rcStrict == VINF_SUCCESS)
11238 {
11239 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11240 AssertRCReturn(rc, rc);
11241 }
11242 else
11243 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11244 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11245 || RT_FAILURE(rcStrict));
11246
11247 /* If the hypercall changes anything other than guest's general-purpose registers,
11248 we would need to reload the guest changed bits here before VM-entry. */
11249 }
11250 else
11251 Log4Func(("Hypercalls not enabled\n"));
11252
11253 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11254 if (RT_FAILURE(rcStrict))
11255 {
11256 hmR0VmxSetPendingXcptUD(pVCpu);
11257 rcStrict = VINF_SUCCESS;
11258 }
11259
11260 return rcStrict;
11261}
11262
11263
11264/**
11265 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11266 */
11267HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11268{
11269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11270 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11271
11272 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11273 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11274 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11275 AssertRCReturn(rc, rc);
11276
11277 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11278
11279 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11280 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11281 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11282 {
11283 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11284 rcStrict = VINF_SUCCESS;
11285 }
11286 else
11287 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11288 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11289 return rcStrict;
11290}
11291
11292
11293/**
11294 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11295 */
11296HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11297{
11298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11299 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11300 AssertRCReturn(rc, rc);
11301
11302 PVM pVM = pVCpu->CTX_SUFF(pVM);
11303 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11304 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11305 if (RT_LIKELY(rc == VINF_SUCCESS))
11306 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11307 else
11308 {
11309 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11310 rc = VERR_EM_INTERPRETER;
11311 }
11312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11313 return rc;
11314}
11315
11316
11317/**
11318 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11319 */
11320HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11321{
11322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11323 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11324 AssertRCReturn(rc, rc);
11325
11326 PVM pVM = pVCpu->CTX_SUFF(pVM);
11327 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11328 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11329 rc = VBOXSTRICTRC_VAL(rc2);
11330 if (RT_LIKELY( rc == VINF_SUCCESS
11331 || rc == VINF_EM_HALT))
11332 {
11333 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11334 AssertRCReturn(rc3, rc3);
11335
11336 if ( rc == VINF_EM_HALT
11337 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11338 rc = VINF_SUCCESS;
11339 }
11340 else
11341 {
11342 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11343 rc = VERR_EM_INTERPRETER;
11344 }
11345 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11346 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11348 return rc;
11349}
11350
11351
11352/**
11353 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11354 */
11355HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11356{
11357 /*
11358 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11359 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11360 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11361 * VMX root operation. If we get here, something funny is going on.
11362 *
11363 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11364 */
11365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11366 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11367 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11368}
11369
11370
11371/**
11372 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11373 */
11374HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11375{
11376 /*
11377 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11378 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11379 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11380 * an SMI. If we get here, something funny is going on.
11381 *
11382 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11383 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11384 */
11385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11386 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11387 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11388}
11389
11390
11391/**
11392 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11393 */
11394HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11395{
11396 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11397 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11398 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11399 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11400}
11401
11402
11403/**
11404 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11405 */
11406HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11407{
11408 /*
11409 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11410 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11411 * See Intel spec. 25.3 "Other Causes of VM-exits".
11412 */
11413 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11414 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11415 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11416}
11417
11418
11419/**
11420 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11421 * VM-exit.
11422 */
11423HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11424{
11425 /*
11426 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11427 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11428 *
11429 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11430 * See Intel spec. "23.8 Restrictions on VMX operation".
11431 */
11432 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11433 return VINF_SUCCESS;
11434}
11435
11436
11437/**
11438 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11439 * VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11444 return VINF_EM_RESET;
11445}
11446
11447
11448/**
11449 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11450 */
11451HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11452{
11453 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11454 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11455
11456 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11457 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11458 AssertRCReturn(rc, rc);
11459
11460 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11461 rc = VINF_SUCCESS;
11462 else
11463 rc = VINF_EM_HALT;
11464
11465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11466 if (rc != VINF_SUCCESS)
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11468 return rc;
11469}
11470
11471
11472/**
11473 * VM-exit handler for instructions that result in a \#UD exception delivered to
11474 * the guest.
11475 */
11476HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11477{
11478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11479 hmR0VmxSetPendingXcptUD(pVCpu);
11480 return VINF_SUCCESS;
11481}
11482
11483
11484/**
11485 * VM-exit handler for expiry of the VMX preemption timer.
11486 */
11487HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11488{
11489 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11490
11491 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11492 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11493
11494 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11495 PVM pVM = pVCpu->CTX_SUFF(pVM);
11496 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11498 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11499}
11500
11501
11502/**
11503 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11504 */
11505HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11506{
11507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11508
11509 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11510 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11511 AssertRCReturn(rc, rc);
11512
11513 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11514 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11515 : HM_CHANGED_XCPT_RAISED_MASK);
11516
11517 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11518 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11519
11520 return rcStrict;
11521}
11522
11523
11524/**
11525 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11526 */
11527HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11528{
11529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11530 /** @todo Use VM-exit instruction information. */
11531 return VERR_EM_INTERPRETER;
11532}
11533
11534
11535/**
11536 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11537 * Error VM-exit.
11538 */
11539HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11540{
11541 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11542 AssertRCReturn(rc, rc);
11543 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11544 if (RT_FAILURE(rc))
11545 return rc;
11546
11547 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11548 NOREF(uInvalidReason);
11549
11550#ifdef VBOX_STRICT
11551 uint32_t fIntrState;
11552 RTHCUINTREG uHCReg;
11553 uint64_t u64Val;
11554 uint32_t u32Val;
11555
11556 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11557 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11558 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11559 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11560 AssertRCReturn(rc, rc);
11561
11562 Log4(("uInvalidReason %u\n", uInvalidReason));
11563 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11564 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11565 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11566 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11567
11568 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11569 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11570 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11571 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11572 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11573 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11574 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11575 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11576 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11577 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11578 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11579 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11580
11581 hmR0DumpRegs(pVCpu);
11582#else
11583 NOREF(pVmxTransient);
11584#endif
11585
11586 return VERR_VMX_INVALID_GUEST_STATE;
11587}
11588
11589
11590/**
11591 * VM-exit handler for VM-entry failure due to an MSR-load
11592 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11593 */
11594HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11595{
11596 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11597 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11598}
11599
11600
11601/**
11602 * VM-exit handler for VM-entry failure due to a machine-check event
11603 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11604 */
11605HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11606{
11607 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11608 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11609}
11610
11611
11612/**
11613 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11614 * theory.
11615 */
11616HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11617{
11618 RT_NOREF2(pVCpu, pVmxTransient);
11619 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11620 return VERR_VMX_UNDEFINED_EXIT_CODE;
11621}
11622
11623
11624/**
11625 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11626 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11627 * Conditional VM-exit.
11628 */
11629HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11630{
11631 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11632
11633 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11635 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11636 return VERR_EM_INTERPRETER;
11637 AssertMsgFailed(("Unexpected XDTR access\n"));
11638 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11639}
11640
11641
11642/**
11643 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11644 */
11645HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11646{
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11648
11649 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11650 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11651 return VERR_EM_INTERPRETER;
11652 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11653 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11654}
11655
11656
11657/**
11658 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11659 */
11660HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11661{
11662 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11663
11664 /** @todo Optimize this: We currently drag in in the whole MSR state
11665 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11666 * MSRs required. That would require changes to IEM and possibly CPUM too.
11667 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11668 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11669 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11670 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11671 AssertRCReturn(rc, rc);
11672
11673 Log4Func(("ecx=%#RX32\n", idMsr));
11674
11675#ifdef VBOX_STRICT
11676 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11677 {
11678 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11679 && idMsr != MSR_K6_EFER)
11680 {
11681 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11682 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11683 }
11684 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11685 {
11686 VMXMSREXITREAD enmRead;
11687 VMXMSREXITWRITE enmWrite;
11688 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11689 AssertRCReturn(rc2, rc2);
11690 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11691 {
11692 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11693 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11694 }
11695 }
11696 }
11697#endif
11698
11699 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11701 if (rcStrict == VINF_SUCCESS)
11702 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11703 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11704 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11705 {
11706 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11707 rcStrict = VINF_SUCCESS;
11708 }
11709 else
11710 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11711
11712 return rcStrict;
11713}
11714
11715
11716/**
11717 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11718 */
11719HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11722
11723 /** @todo Optimize this: We currently drag in in the whole MSR state
11724 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11725 * MSRs required. That would require changes to IEM and possibly CPUM too.
11726 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11727 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11728 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11729 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11730 AssertRCReturn(rc, rc);
11731
11732 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11733
11734 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11736
11737 if (rcStrict == VINF_SUCCESS)
11738 {
11739 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11740
11741 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11742 if ( idMsr == MSR_IA32_APICBASE
11743 || ( idMsr >= MSR_IA32_X2APIC_START
11744 && idMsr <= MSR_IA32_X2APIC_END))
11745 {
11746 /*
11747 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11748 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11749 */
11750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11751 }
11752 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11753 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11754 else if (idMsr == MSR_K6_EFER)
11755 {
11756 /*
11757 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11758 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11759 * the other bits as well, SCE and NXE. See @bugref{7368}.
11760 */
11761 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11762 | HM_CHANGED_VMX_EXIT_CTLS);
11763 }
11764
11765 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11766 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11767 {
11768 switch (idMsr)
11769 {
11770 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11771 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11772 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11773 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11774 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11775 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11776 default:
11777 {
11778 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11779 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11780 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11781 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11782 break;
11783 }
11784 }
11785 }
11786#ifdef VBOX_STRICT
11787 else
11788 {
11789 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11790 switch (idMsr)
11791 {
11792 case MSR_IA32_SYSENTER_CS:
11793 case MSR_IA32_SYSENTER_EIP:
11794 case MSR_IA32_SYSENTER_ESP:
11795 case MSR_K8_FS_BASE:
11796 case MSR_K8_GS_BASE:
11797 {
11798 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11799 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11800 }
11801
11802 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11803 default:
11804 {
11805 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11806 {
11807 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11808 if (idMsr != MSR_K6_EFER)
11809 {
11810 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11811 idMsr));
11812 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11813 }
11814 }
11815
11816 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11817 {
11818 VMXMSREXITREAD enmRead;
11819 VMXMSREXITWRITE enmWrite;
11820 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11821 AssertRCReturn(rc2, rc2);
11822 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11823 {
11824 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11825 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11826 }
11827 }
11828 break;
11829 }
11830 }
11831 }
11832#endif /* VBOX_STRICT */
11833 }
11834 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11835 {
11836 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11837 rcStrict = VINF_SUCCESS;
11838 }
11839 else
11840 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11841
11842 return rcStrict;
11843}
11844
11845
11846/**
11847 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11848 */
11849HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11850{
11851 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11852 /** @todo The guest has likely hit a contended spinlock. We might want to
11853 * poke a schedule different guest VCPU. */
11854 return VINF_EM_RAW_INTERRUPT;
11855}
11856
11857
11858/**
11859 * VM-exit handler for when the TPR value is lowered below the specified
11860 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11861 */
11862HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11863{
11864 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11865 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11866
11867 /*
11868 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11869 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11870 */
11871 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11872 return VINF_SUCCESS;
11873}
11874
11875
11876/**
11877 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11878 * VM-exit.
11879 *
11880 * @retval VINF_SUCCESS when guest execution can continue.
11881 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11882 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11883 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11884 * interpreter.
11885 */
11886HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11887{
11888 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11889 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11890
11891 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11892 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11893 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11894 AssertRCReturn(rc, rc);
11895
11896 VBOXSTRICTRC rcStrict;
11897 PVM pVM = pVCpu->CTX_SUFF(pVM);
11898 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11899 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11900 switch (uAccessType)
11901 {
11902 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11903 {
11904 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11905 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11906 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11907 AssertMsg( rcStrict == VINF_SUCCESS
11908 || rcStrict == VINF_IEM_RAISED_XCPT
11909 || rcStrict == VINF_PGM_CHANGE_MODE
11910 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11911
11912 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11913 {
11914 case 0:
11915 {
11916 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11917 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11919 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11920 break;
11921 }
11922
11923 case 2:
11924 {
11925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11926 /* Nothing to do here, CR2 it's not part of the VMCS. */
11927 break;
11928 }
11929
11930 case 3:
11931 {
11932 Assert( !pVM->hm.s.fNestedPaging
11933 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11934 || pVCpu->hm.s.fUsingDebugLoop);
11935 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11936 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11937 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11938 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11939 break;
11940 }
11941
11942 case 4:
11943 {
11944 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11945 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11946 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11947 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11948 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11949 break;
11950 }
11951
11952 case 8:
11953 {
11954 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11955 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11956 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11957 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11958 break;
11959 }
11960 default:
11961 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11962 break;
11963 }
11964 break;
11965 }
11966
11967 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11968 {
11969 Assert( !pVM->hm.s.fNestedPaging
11970 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11971 || pVCpu->hm.s.fUsingDebugLoop
11972 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11973 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11974 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11975 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11976
11977 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11978 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11979 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11980 AssertMsg( rcStrict == VINF_SUCCESS
11981 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11982#ifdef VBOX_WITH_STATISTICS
11983 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11984 {
11985 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11986 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11987 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11988 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11989 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11990 }
11991#endif
11992 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11993 VBOXSTRICTRC_VAL(rcStrict)));
11994 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
11995 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
11996 else
11997 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11998 break;
11999 }
12000
12001 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12002 {
12003 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12004 AssertMsg( rcStrict == VINF_SUCCESS
12005 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12006
12007 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12009 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12010 break;
12011 }
12012
12013 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12014 {
12015 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12016 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12017 AssertMsg( rcStrict == VINF_SUCCESS
12018 || rcStrict == VINF_IEM_RAISED_XCPT
12019 || rcStrict == VINF_PGM_CHANGE_MODE,
12020 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12021
12022 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12023 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12024 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12025 break;
12026 }
12027
12028 default:
12029 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12030 VERR_VMX_UNEXPECTED_EXCEPTION);
12031 }
12032
12033 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12034 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12035 if (rcStrict == VINF_IEM_RAISED_XCPT)
12036 {
12037 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
12038 rcStrict = VINF_SUCCESS;
12039 }
12040
12041 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12042 NOREF(pVM);
12043 return rcStrict;
12044}
12045
12046
12047/**
12048 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12049 * VM-exit.
12050 */
12051HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12052{
12053 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12054 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12055
12056 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12057 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12058 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12059 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12060 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12061 AssertRCReturn(rc, rc);
12062
12063 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12064 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12065 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12066 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12067 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12068 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12069 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12070 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12071 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12072
12073 /*
12074 * Update exit history to see if this exit can be optimized.
12075 */
12076 VBOXSTRICTRC rcStrict;
12077 PCEMEXITREC pExitRec = NULL;
12078 if ( !fGstStepping
12079 && !fDbgStepping)
12080 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12081 !fIOString
12082 ? !fIOWrite
12083 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12084 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12085 : !fIOWrite
12086 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12087 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12088 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12089 if (!pExitRec)
12090 {
12091 /* I/O operation lookup arrays. */
12092 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12093 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12094 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12095 uint32_t const cbInstr = pVmxTransient->cbInstr;
12096 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12097 PVM pVM = pVCpu->CTX_SUFF(pVM);
12098 if (fIOString)
12099 {
12100 /*
12101 * INS/OUTS - I/O String instruction.
12102 *
12103 * Use instruction-information if available, otherwise fall back on
12104 * interpreting the instruction.
12105 */
12106 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12107 fIOWrite ? 'w' : 'r'));
12108 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12109 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12110 {
12111 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12112 AssertRCReturn(rc2, rc2);
12113 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12114 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12115 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12116 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12117 if (fIOWrite)
12118 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12119 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12120 else
12121 {
12122 /*
12123 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12124 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12125 * See Intel Instruction spec. for "INS".
12126 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12127 */
12128 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12129 }
12130 }
12131 else
12132 rcStrict = IEMExecOne(pVCpu);
12133
12134 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12135 fUpdateRipAlready = true;
12136 }
12137 else
12138 {
12139 /*
12140 * IN/OUT - I/O instruction.
12141 */
12142 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12143 fIOWrite ? 'w' : 'r'));
12144 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12145 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12146 if (fIOWrite)
12147 {
12148 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12149 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12150 }
12151 else
12152 {
12153 uint32_t u32Result = 0;
12154 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12155 if (IOM_SUCCESS(rcStrict))
12156 {
12157 /* Save result of I/O IN instr. in AL/AX/EAX. */
12158 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12159 }
12160 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12161 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12162 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12163 }
12164 }
12165
12166 if (IOM_SUCCESS(rcStrict))
12167 {
12168 if (!fUpdateRipAlready)
12169 {
12170 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12171 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12172 }
12173
12174 /*
12175 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12176 * while booting Fedora 17 64-bit guest.
12177 *
12178 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12179 */
12180 if (fIOString)
12181 {
12182 /** @todo Single-step for INS/OUTS with REP prefix? */
12183 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12184 }
12185 else if ( !fDbgStepping
12186 && fGstStepping)
12187 {
12188 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12189 AssertRCReturn(rc, rc);
12190 }
12191
12192 /*
12193 * If any I/O breakpoints are armed, we need to check if one triggered
12194 * and take appropriate action.
12195 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12196 */
12197 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12198 AssertRCReturn(rc, rc);
12199
12200 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12201 * execution engines about whether hyper BPs and such are pending. */
12202 uint32_t const uDr7 = pCtx->dr[7];
12203 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12204 && X86_DR7_ANY_RW_IO(uDr7)
12205 && (pCtx->cr4 & X86_CR4_DE))
12206 || DBGFBpIsHwIoArmed(pVM)))
12207 {
12208 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12209
12210 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12211 VMMRZCallRing3Disable(pVCpu);
12212 HM_DISABLE_PREEMPT(pVCpu);
12213
12214 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12215
12216 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12217 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12218 {
12219 /* Raise #DB. */
12220 if (fIsGuestDbgActive)
12221 ASMSetDR6(pCtx->dr[6]);
12222 if (pCtx->dr[7] != uDr7)
12223 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12224
12225 hmR0VmxSetPendingXcptDB(pVCpu);
12226 }
12227 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12228 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12229 else if ( rcStrict2 != VINF_SUCCESS
12230 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12231 rcStrict = rcStrict2;
12232 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12233
12234 HM_RESTORE_PREEMPT();
12235 VMMRZCallRing3Enable(pVCpu);
12236 }
12237 }
12238
12239#ifdef VBOX_STRICT
12240 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12241 Assert(!fIOWrite);
12242 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12243 Assert(fIOWrite);
12244 else
12245 {
12246# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12247 * statuses, that the VMM device and some others may return. See
12248 * IOM_SUCCESS() for guidance. */
12249 AssertMsg( RT_FAILURE(rcStrict)
12250 || rcStrict == VINF_SUCCESS
12251 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12252 || rcStrict == VINF_EM_DBG_BREAKPOINT
12253 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12254 || rcStrict == VINF_EM_RAW_TO_R3
12255 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12256# endif
12257 }
12258#endif
12259 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12260 }
12261 else
12262 {
12263 /*
12264 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12265 */
12266 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12267 AssertRCReturn(rc2, rc2);
12268 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12269 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12270 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12271 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12272 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12273 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12274
12275 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12276 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12277
12278 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12279 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12280 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12281 }
12282 return rcStrict;
12283}
12284
12285
12286/**
12287 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12288 * VM-exit.
12289 */
12290HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12291{
12292 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12293
12294 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12295 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12296 AssertRCReturn(rc, rc);
12297 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12298 {
12299 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12300 AssertRCReturn(rc, rc);
12301 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12302 {
12303 uint32_t uErrCode;
12304 RTGCUINTPTR GCPtrFaultAddress;
12305 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12306 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12307 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12308 if (fErrorCodeValid)
12309 {
12310 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12311 AssertRCReturn(rc, rc);
12312 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12313 }
12314 else
12315 uErrCode = 0;
12316
12317 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12318 && uVector == X86_XCPT_PF)
12319 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12320 else
12321 GCPtrFaultAddress = 0;
12322
12323 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12324 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12325
12326 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12327 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12328 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12329 }
12330 }
12331
12332 /* Fall back to the interpreter to emulate the task-switch. */
12333 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12334 return VERR_EM_INTERPRETER;
12335}
12336
12337
12338/**
12339 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12340 */
12341HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12342{
12343 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12344 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12345 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12346 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12347 AssertRCReturn(rc, rc);
12348 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12349 return VINF_EM_DBG_STEPPED;
12350}
12351
12352
12353/**
12354 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12355 */
12356HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12357{
12358 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12359
12360 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12361
12362 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12363 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12364 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12365 {
12366 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12367 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12368 {
12369 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12370 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12371 }
12372 }
12373 else
12374 {
12375 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12376 rcStrict1 = VINF_SUCCESS;
12377 return rcStrict1;
12378 }
12379
12380 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12381 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12382 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12383 AssertRCReturn(rc, rc);
12384
12385 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12386 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12387 VBOXSTRICTRC rcStrict2;
12388 switch (uAccessType)
12389 {
12390 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12391 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12392 {
12393 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12394 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12395 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12396
12397 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12398 GCPhys &= PAGE_BASE_GC_MASK;
12399 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12400 PVM pVM = pVCpu->CTX_SUFF(pVM);
12401 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12402 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12403
12404 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12405 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12406 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12407 CPUMCTX2CORE(pCtx), GCPhys);
12408 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12409 if ( rcStrict2 == VINF_SUCCESS
12410 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12411 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12412 {
12413 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12414 | HM_CHANGED_GUEST_APIC_TPR);
12415 rcStrict2 = VINF_SUCCESS;
12416 }
12417 break;
12418 }
12419
12420 default:
12421 Log4Func(("uAccessType=%#x\n", uAccessType));
12422 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12423 break;
12424 }
12425
12426 if (rcStrict2 != VINF_SUCCESS)
12427 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12428 return rcStrict2;
12429}
12430
12431
12432/**
12433 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12434 * VM-exit.
12435 */
12436HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12437{
12438 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12439
12440 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12441 if (pVmxTransient->fWasGuestDebugStateActive)
12442 {
12443 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12444 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12445 }
12446
12447 if ( !pVCpu->hm.s.fSingleInstruction
12448 && !pVmxTransient->fWasHyperDebugStateActive)
12449 {
12450 Assert(!DBGFIsStepping(pVCpu));
12451 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12452
12453 /* Don't intercept MOV DRx any more. */
12454 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12455 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12456 AssertRCReturn(rc, rc);
12457
12458 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12459 VMMRZCallRing3Disable(pVCpu);
12460 HM_DISABLE_PREEMPT(pVCpu);
12461
12462 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12463 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12464 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12465
12466 HM_RESTORE_PREEMPT();
12467 VMMRZCallRing3Enable(pVCpu);
12468
12469#ifdef VBOX_WITH_STATISTICS
12470 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12471 AssertRCReturn(rc, rc);
12472 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12473 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12474 else
12475 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12476#endif
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12478 return VINF_SUCCESS;
12479 }
12480
12481 /*
12482 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12483 * Update the segment registers and DR7 from the CPU.
12484 */
12485 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12486 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12487 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12488 AssertRCReturn(rc, rc);
12489 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12490
12491 PVM pVM = pVCpu->CTX_SUFF(pVM);
12492 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12493 {
12494 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12495 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12496 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12497 if (RT_SUCCESS(rc))
12498 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12500 }
12501 else
12502 {
12503 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12504 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12505 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12507 }
12508
12509 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12510 if (RT_SUCCESS(rc))
12511 {
12512 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12513 AssertRCReturn(rc2, rc2);
12514 return VINF_SUCCESS;
12515 }
12516 return rc;
12517}
12518
12519
12520/**
12521 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12522 * Conditional VM-exit.
12523 */
12524HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12525{
12526 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12527 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12528
12529 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12530 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12531 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12532 {
12533 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12534 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12535 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12536 {
12537 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12538 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12539 }
12540 }
12541 else
12542 {
12543 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12544 rcStrict1 = VINF_SUCCESS;
12545 return rcStrict1;
12546 }
12547
12548 /*
12549 * Get sufficent state and update the exit history entry.
12550 */
12551 RTGCPHYS GCPhys;
12552 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12553 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12554 AssertRCReturn(rc, rc);
12555
12556 VBOXSTRICTRC rcStrict;
12557 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12558 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12559 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12560 if (!pExitRec)
12561 {
12562 /*
12563 * If we succeed, resume guest execution.
12564 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12565 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12566 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12567 * weird case. See @bugref{6043}.
12568 */
12569 PVM pVM = pVCpu->CTX_SUFF(pVM);
12570 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12571 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12572 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12573 if ( rcStrict == VINF_SUCCESS
12574 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12575 || rcStrict == VERR_PAGE_NOT_PRESENT)
12576 {
12577 /* Successfully handled MMIO operation. */
12578 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12579 | HM_CHANGED_GUEST_APIC_TPR);
12580 rcStrict = VINF_SUCCESS;
12581 }
12582 }
12583 else
12584 {
12585 /*
12586 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12587 */
12588 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12589 AssertRCReturn(rc2, rc2);
12590
12591 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12592 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12593
12594 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12595 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12596
12597 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12598 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12599 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12600 }
12601 return VBOXSTRICTRC_TODO(rcStrict);
12602}
12603
12604
12605/**
12606 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12607 * VM-exit.
12608 */
12609HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12610{
12611 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12612 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12613
12614 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12615 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12616 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12617 {
12618 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12619 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12620 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12621 }
12622 else
12623 {
12624 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12625 rcStrict1 = VINF_SUCCESS;
12626 return rcStrict1;
12627 }
12628
12629 RTGCPHYS GCPhys;
12630 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12631 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12632 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12633 AssertRCReturn(rc, rc);
12634
12635 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12636 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12637
12638 RTGCUINT uErrorCode = 0;
12639 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12640 uErrorCode |= X86_TRAP_PF_ID;
12641 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12642 uErrorCode |= X86_TRAP_PF_RW;
12643 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12644 uErrorCode |= X86_TRAP_PF_P;
12645
12646 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12647
12648
12649 /* Handle the pagefault trap for the nested shadow table. */
12650 PVM pVM = pVCpu->CTX_SUFF(pVM);
12651 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12652
12653 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12654 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12655
12656 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12657 TRPMResetTrap(pVCpu);
12658
12659 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12660 if ( rcStrict2 == VINF_SUCCESS
12661 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12662 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12663 {
12664 /* Successfully synced our nested page tables. */
12665 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12666 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12667 return VINF_SUCCESS;
12668 }
12669
12670 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12671 return rcStrict2;
12672}
12673
12674/** @} */
12675
12676/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12677/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12678/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12679
12680/** @name VM-exit exception handlers.
12681 * @{
12682 */
12683
12684/**
12685 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12686 */
12687static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12688{
12689 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12690 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12691
12692 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12693 AssertRCReturn(rc, rc);
12694
12695 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12696 {
12697 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12698 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12699
12700 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12701 * provides VM-exit instruction length. If this causes problem later,
12702 * disassemble the instruction like it's done on AMD-V. */
12703 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12704 AssertRCReturn(rc2, rc2);
12705 return rc;
12706 }
12707
12708 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12709 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12710 return rc;
12711}
12712
12713
12714/**
12715 * VM-exit exception handler for \#BP (Breakpoint exception).
12716 */
12717static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12718{
12719 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12721
12722 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12723 AssertRCReturn(rc, rc);
12724
12725 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12726 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12727 if (rc == VINF_EM_RAW_GUEST_TRAP)
12728 {
12729 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12730 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12731 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12732 AssertRCReturn(rc, rc);
12733
12734 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12735 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12736 }
12737
12738 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12739 return rc;
12740}
12741
12742
12743/**
12744 * VM-exit exception handler for \#AC (alignment check exception).
12745 */
12746static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12747{
12748 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12749
12750 /*
12751 * Re-inject it. We'll detect any nesting before getting here.
12752 */
12753 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12754 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12755 AssertRCReturn(rc, rc);
12756 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12757
12758 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12759 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12760 return VINF_SUCCESS;
12761}
12762
12763
12764/**
12765 * VM-exit exception handler for \#DB (Debug exception).
12766 */
12767static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12768{
12769 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12770 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12771
12772 /*
12773 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12774 * for processing.
12775 */
12776 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12777
12778 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12779 uint64_t uDR6 = X86_DR6_INIT_VAL;
12780 uDR6 |= ( pVmxTransient->uExitQualification
12781 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12782
12783 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12784 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12785 Log6Func(("rc=%Rrc\n", rc));
12786 if (rc == VINF_EM_RAW_GUEST_TRAP)
12787 {
12788 /*
12789 * The exception was for the guest. Update DR6, DR7.GD and
12790 * IA32_DEBUGCTL.LBR before forwarding it.
12791 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12792 */
12793 VMMRZCallRing3Disable(pVCpu);
12794 HM_DISABLE_PREEMPT(pVCpu);
12795
12796 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12797 pCtx->dr[6] |= uDR6;
12798 if (CPUMIsGuestDebugStateActive(pVCpu))
12799 ASMSetDR6(pCtx->dr[6]);
12800
12801 HM_RESTORE_PREEMPT();
12802 VMMRZCallRing3Enable(pVCpu);
12803
12804 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12805 AssertRCReturn(rc, rc);
12806
12807 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12808 pCtx->dr[7] &= ~X86_DR7_GD;
12809
12810 /* Paranoia. */
12811 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12812 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12813
12814 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12815 AssertRCReturn(rc, rc);
12816
12817 /*
12818 * Raise #DB in the guest.
12819 *
12820 * It is important to reflect exactly what the VM-exit gave us (preserving the
12821 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12822 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12823 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12824 *
12825 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12826 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12827 */
12828 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12829 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12830 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12831 AssertRCReturn(rc, rc);
12832 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12833 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12834 return VINF_SUCCESS;
12835 }
12836
12837 /*
12838 * Not a guest trap, must be a hypervisor related debug event then.
12839 * Update DR6 in case someone is interested in it.
12840 */
12841 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12842 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12843 CPUMSetHyperDR6(pVCpu, uDR6);
12844
12845 return rc;
12846}
12847
12848/**
12849 * VM-exit exception handler for \#GP (General-protection exception).
12850 *
12851 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12852 */
12853static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12854{
12855 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12857
12858 int rc;
12859 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12860 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12861 { /* likely */ }
12862 else
12863 {
12864#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12865 Assert(pVCpu->hm.s.fUsingDebugLoop);
12866#endif
12867 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12868 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12869 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12870 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12871 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12872 AssertRCReturn(rc, rc);
12873 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12874 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12875 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12876 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12877 return rc;
12878 }
12879
12880 Assert(CPUMIsGuestInRealModeEx(pCtx));
12881 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12882
12883 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12884 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12885 AssertRCReturn(rc, rc);
12886
12887 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12888 uint32_t cbOp = 0;
12889 PVM pVM = pVCpu->CTX_SUFF(pVM);
12890 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12891 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12892 if (RT_SUCCESS(rc))
12893 {
12894 rc = VINF_SUCCESS;
12895 Assert(cbOp == pDis->cbInstr);
12896 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12897 switch (pDis->pCurInstr->uOpcode)
12898 {
12899 case OP_CLI:
12900 {
12901 pCtx->eflags.Bits.u1IF = 0;
12902 pCtx->eflags.Bits.u1RF = 0;
12903 pCtx->rip += pDis->cbInstr;
12904 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12905 if ( !fDbgStepping
12906 && pCtx->eflags.Bits.u1TF)
12907 {
12908 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12909 AssertRCReturn(rc, rc);
12910 }
12911 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12912 break;
12913 }
12914
12915 case OP_STI:
12916 {
12917 bool fOldIF = pCtx->eflags.Bits.u1IF;
12918 pCtx->eflags.Bits.u1IF = 1;
12919 pCtx->eflags.Bits.u1RF = 0;
12920 pCtx->rip += pDis->cbInstr;
12921 if (!fOldIF)
12922 {
12923 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12924 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12925 }
12926 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12927 if ( !fDbgStepping
12928 && pCtx->eflags.Bits.u1TF)
12929 {
12930 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12931 AssertRCReturn(rc, rc);
12932 }
12933 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12934 break;
12935 }
12936
12937 case OP_HLT:
12938 {
12939 rc = VINF_EM_HALT;
12940 pCtx->rip += pDis->cbInstr;
12941 pCtx->eflags.Bits.u1RF = 0;
12942 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12943 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12944 break;
12945 }
12946
12947 case OP_POPF:
12948 {
12949 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12950 uint32_t cbParm;
12951 uint32_t uMask;
12952 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12953 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12954 {
12955 cbParm = 4;
12956 uMask = 0xffffffff;
12957 }
12958 else
12959 {
12960 cbParm = 2;
12961 uMask = 0xffff;
12962 }
12963
12964 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12965 RTGCPTR GCPtrStack = 0;
12966 X86EFLAGS Eflags;
12967 Eflags.u32 = 0;
12968 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12969 &GCPtrStack);
12970 if (RT_SUCCESS(rc))
12971 {
12972 Assert(sizeof(Eflags.u32) >= cbParm);
12973 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12974 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12975 }
12976 if (RT_FAILURE(rc))
12977 {
12978 rc = VERR_EM_INTERPRETER;
12979 break;
12980 }
12981 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
12982 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
12983 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12984 pCtx->esp += cbParm;
12985 pCtx->esp &= uMask;
12986 pCtx->rip += pDis->cbInstr;
12987 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12988 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12989 POPF restores EFLAGS.TF. */
12990 if ( !fDbgStepping
12991 && fGstStepping)
12992 {
12993 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12994 AssertRCReturn(rc, rc);
12995 }
12996 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
12997 break;
12998 }
12999
13000 case OP_PUSHF:
13001 {
13002 uint32_t cbParm;
13003 uint32_t uMask;
13004 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13005 {
13006 cbParm = 4;
13007 uMask = 0xffffffff;
13008 }
13009 else
13010 {
13011 cbParm = 2;
13012 uMask = 0xffff;
13013 }
13014
13015 /* Get the stack pointer & push the contents of eflags onto the stack. */
13016 RTGCPTR GCPtrStack = 0;
13017 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13018 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13019 if (RT_FAILURE(rc))
13020 {
13021 rc = VERR_EM_INTERPRETER;
13022 break;
13023 }
13024 X86EFLAGS Eflags = pCtx->eflags;
13025 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13026 Eflags.Bits.u1RF = 0;
13027 Eflags.Bits.u1VM = 0;
13028
13029 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13030 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13031 {
13032 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13033 rc = VERR_EM_INTERPRETER;
13034 break;
13035 }
13036 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13037 pCtx->esp -= cbParm;
13038 pCtx->esp &= uMask;
13039 pCtx->rip += pDis->cbInstr;
13040 pCtx->eflags.Bits.u1RF = 0;
13041 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13042 if ( !fDbgStepping
13043 && pCtx->eflags.Bits.u1TF)
13044 {
13045 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13046 AssertRCReturn(rc, rc);
13047 }
13048 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13049 break;
13050 }
13051
13052 case OP_IRET:
13053 {
13054 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13055 * instruction reference. */
13056 RTGCPTR GCPtrStack = 0;
13057 uint32_t uMask = 0xffff;
13058 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13059 uint16_t aIretFrame[3];
13060 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13061 {
13062 rc = VERR_EM_INTERPRETER;
13063 break;
13064 }
13065 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13066 &GCPtrStack);
13067 if (RT_SUCCESS(rc))
13068 {
13069 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13070 PGMACCESSORIGIN_HM));
13071 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13072 }
13073 if (RT_FAILURE(rc))
13074 {
13075 rc = VERR_EM_INTERPRETER;
13076 break;
13077 }
13078 pCtx->eip = 0;
13079 pCtx->ip = aIretFrame[0];
13080 pCtx->cs.Sel = aIretFrame[1];
13081 pCtx->cs.ValidSel = aIretFrame[1];
13082 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13083 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13084 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13085 pCtx->sp += sizeof(aIretFrame);
13086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13087 | HM_CHANGED_GUEST_CS);
13088 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13089 if ( !fDbgStepping
13090 && fGstStepping)
13091 {
13092 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13093 AssertRCReturn(rc, rc);
13094 }
13095 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13096 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13097 break;
13098 }
13099
13100 case OP_INT:
13101 {
13102 uint16_t uVector = pDis->Param1.uValue & 0xff;
13103 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13104 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13105 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13106 break;
13107 }
13108
13109 case OP_INTO:
13110 {
13111 if (pCtx->eflags.Bits.u1OF)
13112 {
13113 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13114 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13115 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13116 }
13117 else
13118 {
13119 pCtx->eflags.Bits.u1RF = 0;
13120 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13121 }
13122 break;
13123 }
13124
13125 default:
13126 {
13127 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13128 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13129 EMCODETYPE_SUPERVISOR);
13130 rc = VBOXSTRICTRC_VAL(rc2);
13131 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13132 /** @todo We have to set pending-debug exceptions here when the guest is
13133 * single-stepping depending on the instruction that was interpreted. */
13134 Log4Func(("#GP rc=%Rrc\n", rc));
13135 break;
13136 }
13137 }
13138 }
13139 else
13140 rc = VERR_EM_INTERPRETER;
13141
13142 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13143 ("#GP Unexpected rc=%Rrc\n", rc));
13144 return rc;
13145}
13146
13147
13148/**
13149 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13150 * the exception reported in the VMX transient structure back into the VM.
13151 *
13152 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13153 * up-to-date.
13154 */
13155static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13156{
13157 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13158#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13159 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13160 ("uVector=%#x u32XcptBitmap=%#X32\n",
13161 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13162#endif
13163
13164 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13165 hmR0VmxCheckExitDueToEventDelivery(). */
13166 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13167 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13168 AssertRCReturn(rc, rc);
13169 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13170
13171#ifdef DEBUG_ramshankar
13172 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13173 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13174 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13175#endif
13176
13177 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13178 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13179 return VINF_SUCCESS;
13180}
13181
13182
13183/**
13184 * VM-exit exception handler for \#PF (Page-fault exception).
13185 */
13186static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13187{
13188 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13189 PVM pVM = pVCpu->CTX_SUFF(pVM);
13190 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13191 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13192 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13193 AssertRCReturn(rc, rc);
13194
13195 if (!pVM->hm.s.fNestedPaging)
13196 { /* likely */ }
13197 else
13198 {
13199#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13200 Assert(pVCpu->hm.s.fUsingDebugLoop);
13201#endif
13202 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13203 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13204 {
13205 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13206 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13207 }
13208 else
13209 {
13210 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13211 hmR0VmxSetPendingXcptDF(pVCpu);
13212 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13213 }
13214 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13215 return rc;
13216 }
13217
13218 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13219 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13220 if (pVmxTransient->fVectoringPF)
13221 {
13222 Assert(pVCpu->hm.s.Event.fPending);
13223 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13224 }
13225
13226 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13227 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13228 AssertRCReturn(rc, rc);
13229
13230 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13231 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13232
13233 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13234 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13235 (RTGCPTR)pVmxTransient->uExitQualification);
13236
13237 Log4Func(("#PF: rc=%Rrc\n", rc));
13238 if (rc == VINF_SUCCESS)
13239 {
13240 /*
13241 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13242 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13243 */
13244 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13245 TRPMResetTrap(pVCpu);
13246 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13247 return rc;
13248 }
13249
13250 if (rc == VINF_EM_RAW_GUEST_TRAP)
13251 {
13252 if (!pVmxTransient->fVectoringDoublePF)
13253 {
13254 /* It's a guest page fault and needs to be reflected to the guest. */
13255 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13256 TRPMResetTrap(pVCpu);
13257 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13258 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13259 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13260 }
13261 else
13262 {
13263 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13264 TRPMResetTrap(pVCpu);
13265 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13266 hmR0VmxSetPendingXcptDF(pVCpu);
13267 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13268 }
13269
13270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13271 return VINF_SUCCESS;
13272 }
13273
13274 TRPMResetTrap(pVCpu);
13275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13276 return rc;
13277}
13278
13279/** @} */
13280
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette