VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72995

Last change on this file since 72995 was 72995, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Attempted build fix.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 563.3 KB
Line 
1/* $Id: HMVMXR0.cpp 72995 2018-07-09 04:36:20Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Loads a set of guests MSRs to allow read/passthru to the guest.
1524 *
1525 * The name of this function is slightly confusing. This function does NOT
1526 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527 * common prefix for functions dealing with "lazy restoration" of the shared
1528 * MSRs.
1529 *
1530 * @param pVCpu The cross context virtual CPU structure.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 /*
1544 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546 * we can skip a few MSR writes.
1547 *
1548 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549 * guest MSR values in the guest-CPU context might be different to what's currently
1550 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551 * CPU, see @bugref{8728}.
1552 */
1553 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559 {
1560#ifdef VBOX_STRICT
1561 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565#endif
1566 }
1567 else
1568 {
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573 }
1574 }
1575#endif
1576 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxImportGuestState()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598#if HC_ARCH_BITS == 64
1599 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600 {
1601 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605 }
1606#endif
1607 }
1608 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1609}
1610
1611
1612/**
1613 * Verifies that our cached values of the VMCS fields are all consistent with
1614 * what's actually present in the VMCS.
1615 *
1616 * @returns VBox status code.
1617 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619 * VMCS content. HMCPU error-field is
1620 * updated, see VMX_VCI_XXX.
1621 * @param pVCpu The cross context virtual CPU structure.
1622 */
1623static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624{
1625 uint32_t u32Val;
1626 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627 AssertRCReturn(rc, rc);
1628 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655 {
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662 }
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671 uint64_t u64Val;
1672 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673 AssertRCReturn(rc, rc);
1674 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679 return VINF_SUCCESS;
1680}
1681
1682
1683#ifdef VBOX_STRICT
1684/**
1685 * Verifies that our cached host EFER value has not changed
1686 * since we cached it.
1687 *
1688 * @param pVCpu The cross context virtual CPU structure.
1689 */
1690static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691{
1692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695 {
1696 uint64_t u64Val;
1697 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698 AssertRC(rc);
1699
1700 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702 }
1703}
1704
1705
1706/**
1707 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708 * VMCS are correct.
1709 *
1710 * @param pVCpu The cross context virtual CPU structure.
1711 */
1712static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713{
1714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716 /* Verify MSR counts in the VMCS are what we think it should be. */
1717 uint32_t cMsrs;
1718 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730 {
1731 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733 pGuestMsr->u32Msr, cMsrs));
1734
1735 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739 /* Verify that the permissions are as expected in the MSR bitmap. */
1740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741 {
1742 VMXMSREXITREAD enmRead;
1743 VMXMSREXITWRITE enmWrite;
1744 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747 {
1748 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750 }
1751 else
1752 {
1753 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756 pGuestMsr->u32Msr, cMsrs));
1757 }
1758 }
1759 }
1760}
1761#endif /* VBOX_STRICT */
1762
1763
1764/**
1765 * Flushes the TLB using EPT.
1766 *
1767 * @returns VBox status code.
1768 * @param pVCpu The cross context virtual CPU structure of the calling
1769 * EMT. Can be NULL depending on @a enmTlbFlush.
1770 * @param enmTlbFlush Type of flush.
1771 *
1772 * @remarks Caller is responsible for making sure this function is called only
1773 * when NestedPaging is supported and providing @a enmTlbFlush that is
1774 * supported by the CPU.
1775 * @remarks Can be called with interrupts disabled.
1776 */
1777static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778{
1779 uint64_t au64Descriptor[2];
1780 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781 au64Descriptor[0] = 0;
1782 else
1783 {
1784 Assert(pVCpu);
1785 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786 }
1787 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790 AssertMsg(rc == VINF_SUCCESS,
1791 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793 if ( RT_SUCCESS(rc)
1794 && pVCpu)
1795 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796}
1797
1798
1799/**
1800 * Flushes the TLB using VPID.
1801 *
1802 * @returns VBox status code.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmTlbFlush.
1805 * @param enmTlbFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmTlbFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812{
1813 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834 if ( RT_SUCCESS(rc)
1835 && pVCpu)
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 NOREF(rc);
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVCpu The cross context virtual CPU structure.
1847 * @param GCVirt Guest virtual address of the page to invalidate.
1848 */
1849VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850{
1851 AssertPtr(pVCpu);
1852 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 if (!fFlushPending)
1856 {
1857 /*
1858 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859 * the EPT case. See @bugref{6043} and @bugref{6177}.
1860 *
1861 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862 * as this function maybe called in a loop with individual addresses.
1863 */
1864 PVM pVM = pVCpu->CTX_SUFF(pVM);
1865 if (pVM->hm.s.vmx.fVpid)
1866 {
1867 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870 /*
1871 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872 * where executing INVVPID outside 64-bit mode does not flush translations of
1873 * 64-bit linear addresses, see @bugref{6208#c72}.
1874 */
1875 if (RT_HI_U32(GCVirt))
1876 fVpidFlush = false;
1877#endif
1878
1879 if (fVpidFlush)
1880 {
1881 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883 }
1884 else
1885 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886 }
1887 else if (pVM->hm.s.fNestedPaging)
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897 * case where neither EPT nor VPID is supported by the CPU.
1898 *
1899 * @param pVCpu The cross context virtual CPU structure.
1900 * @param pCpu Pointer to the global HM struct.
1901 *
1902 * @remarks Called with interrupts disabled.
1903 */
1904static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905{
1906 AssertPtr(pVCpu);
1907 AssertPtr(pCpu);
1908
1909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914 pVCpu->hm.s.fForceTLBFlush = false;
1915 return;
1916}
1917
1918
1919/**
1920 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921 *
1922 * @param pVCpu The cross context virtual CPU structure.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926 * nomenclature. The reason is, to avoid confusion in compare statements
1927 * since the host-CPU copies are named "ASID".
1928 *
1929 * @remarks Called with interrupts disabled.
1930 */
1931static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932{
1933#ifdef VBOX_WITH_STATISTICS
1934 bool fTlbFlushed = false;
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937 if (!fTlbFlushed) \
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939 } while (0)
1940#else
1941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943#endif
1944
1945 AssertPtr(pCpu);
1946 AssertPtr(pVCpu);
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949 PVM pVM = pVCpu->CTX_SUFF(pVM);
1950 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954 /*
1955 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958 * cannot reuse the current ASID anymore.
1959 */
1960 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962 {
1963 ++pCpu->uCurrentAsid;
1964 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965 {
1966 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969 }
1970
1971 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975 /*
1976 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978 */
1979 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981 HMVMX_SET_TAGGED_TLB_FLUSHED();
1982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1983 }
1984
1985 /* Check for explicit TLB flushes. */
1986 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1987 {
1988 /*
1989 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1990 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1991 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1992 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1993 * mappings, see @bugref{6568}.
1994 *
1995 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1996 */
1997 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1998 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1999 HMVMX_SET_TAGGED_TLB_FLUSHED();
2000 }
2001
2002 pVCpu->hm.s.fForceTLBFlush = false;
2003 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2004
2005 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2006 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2007 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2008 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2009 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2010 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2011 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2012 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2013 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2014
2015 /* Update VMCS with the VPID. */
2016 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2017 AssertRC(rc);
2018
2019#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2020}
2021
2022
2023/**
2024 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2025 *
2026 * @returns VBox status code.
2027 * @param pVCpu The cross context virtual CPU structure.
2028 * @param pCpu Pointer to the global HM CPU struct.
2029 *
2030 * @remarks Called with interrupts disabled.
2031 */
2032static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2033{
2034 AssertPtr(pVCpu);
2035 AssertPtr(pCpu);
2036 Assert(pCpu->idCpu != NIL_RTCPUID);
2037 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2038 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2039
2040 /*
2041 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2042 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2043 */
2044 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2045 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2049 }
2050
2051 /* Check for explicit TLB flushes. */
2052 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2053 {
2054 pVCpu->hm.s.fForceTLBFlush = true;
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2056 }
2057
2058 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2059 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2060
2061 if (pVCpu->hm.s.fForceTLBFlush)
2062 {
2063 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2064 pVCpu->hm.s.fForceTLBFlush = false;
2065 }
2066}
2067
2068
2069/**
2070 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2071 *
2072 * @returns VBox status code.
2073 * @param pVCpu The cross context virtual CPU structure.
2074 * @param pCpu Pointer to the global HM CPU struct.
2075 *
2076 * @remarks Called with interrupts disabled.
2077 */
2078static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2079{
2080 AssertPtr(pVCpu);
2081 AssertPtr(pCpu);
2082 Assert(pCpu->idCpu != NIL_RTCPUID);
2083 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2084 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2085
2086 /*
2087 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2088 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2089 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2090 * cannot reuse the current ASID anymore.
2091 */
2092 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2093 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2094 {
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2097 }
2098
2099 /* Check for explicit TLB flushes. */
2100 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2101 {
2102 /*
2103 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2104 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2105 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2106 * include fExplicitFlush's too) - an obscure corner case.
2107 */
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2110 }
2111
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 ++pCpu->uCurrentAsid;
2117 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2118 {
2119 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2120 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2121 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2122 }
2123
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2126 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2127 if (pCpu->fFlushAsidBeforeUse)
2128 {
2129 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2130 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2131 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2132 {
2133 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2134 pCpu->fFlushAsidBeforeUse = false;
2135 }
2136 else
2137 {
2138 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2139 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2140 }
2141 }
2142 }
2143
2144 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2145 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2146 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2147 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2148 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2149 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2150 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2151
2152 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2153 AssertRC(rc);
2154}
2155
2156
2157/**
2158 * Flushes the guest TLB entry based on CPU capabilities.
2159 *
2160 * @param pVCpu The cross context virtual CPU structure.
2161 * @param pCpu Pointer to the global HM CPU struct.
2162 */
2163DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2164{
2165#ifdef HMVMX_ALWAYS_FLUSH_TLB
2166 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2167#endif
2168 PVM pVM = pVCpu->CTX_SUFF(pVM);
2169 switch (pVM->hm.s.vmx.enmTlbFlushType)
2170 {
2171 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2174 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2175 default:
2176 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2177 break;
2178 }
2179 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2180}
2181
2182
2183/**
2184 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2185 * TLB entries from the host TLB before VM-entry.
2186 *
2187 * @returns VBox status code.
2188 * @param pVM The cross context VM structure.
2189 */
2190static int hmR0VmxSetupTaggedTlb(PVM pVM)
2191{
2192 /*
2193 * Determine optimal flush type for Nested Paging.
2194 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2195 * guest execution (see hmR3InitFinalizeR0()).
2196 */
2197 if (pVM->hm.s.fNestedPaging)
2198 {
2199 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2200 {
2201 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2203 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2204 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2205 else
2206 {
2207 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2208 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2209 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2210 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211 }
2212
2213 /* Make sure the write-back cacheable memory type for EPT is supported. */
2214 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2215 {
2216 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2217 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219 }
2220
2221 /* EPT requires a page-walk length of 4. */
2222 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2223 {
2224 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2225 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2226 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227 }
2228 }
2229 else
2230 {
2231 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2232 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2233 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2234 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235 }
2236 }
2237
2238 /*
2239 * Determine optimal flush type for VPID.
2240 */
2241 if (pVM->hm.s.vmx.fVpid)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2253 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2255 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2256 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2257 pVM->hm.s.vmx.fVpid = false;
2258 }
2259 }
2260 else
2261 {
2262 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2263 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2264 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2265 pVM->hm.s.vmx.fVpid = false;
2266 }
2267 }
2268
2269 /*
2270 * Setup the handler for flushing tagged-TLBs.
2271 */
2272 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2274 else if (pVM->hm.s.fNestedPaging)
2275 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2276 else if (pVM->hm.s.vmx.fVpid)
2277 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2278 else
2279 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2280 return VINF_SUCCESS;
2281}
2282
2283
2284/**
2285 * Sets up pin-based VM-execution controls in the VMCS.
2286 *
2287 * @returns VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure.
2289 *
2290 * @remarks We don't really care about optimizing vmwrites here as it's done only
2291 * once per VM and hence we don't care about VMCS-field cache comparisons.
2292 */
2293static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2294{
2295 PVM pVM = pVCpu->CTX_SUFF(pVM);
2296 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2298
2299 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2300 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2301
2302 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2303 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2304
2305 /* Enable the VMX preemption timer. */
2306 if (pVM->hm.s.vmx.fUsePreemptTimer)
2307 {
2308 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2309 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2310 }
2311
2312#if 0
2313 /* Enable posted-interrupt processing. */
2314 if (pVM->hm.s.fPostedIntrs)
2315 {
2316 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2317 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2318 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2319 }
2320#endif
2321
2322 if ((fVal & fZap) != fVal)
2323 {
2324 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2325 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2328 }
2329
2330 /* Commit it to the VMCS and update our cache. */
2331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2332 AssertRCReturn(rc, rc);
2333 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2334
2335 return VINF_SUCCESS;
2336}
2337
2338
2339/**
2340 * Sets up secondary processor-based VM-execution controls in the VMCS.
2341 *
2342 * @returns VBox status code.
2343 * @param pVCpu The cross context virtual CPU structure.
2344 *
2345 * @remarks We don't really care about optimizing vmwrites here as it's done only
2346 * once per VM and hence we don't care about VMCS-field cache comparisons.
2347 */
2348static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2349{
2350 PVM pVM = pVCpu->CTX_SUFF(pVM);
2351 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354 /* WBINVD causes a VM-exit. */
2355 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2357
2358 /* Enable EPT (aka nested-paging). */
2359 if (pVM->hm.s.fNestedPaging)
2360 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2361
2362 /*
2363 * Enable the INVPCID instruction if supported by the hardware and we expose
2364 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2365 */
2366 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2367 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2369
2370 /* Enable VPID. */
2371 if (pVM->hm.s.vmx.fVpid)
2372 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2373
2374 /* Enable Unrestricted guest execution. */
2375 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2376 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2377
2378#if 0
2379 if (pVM->hm.s.fVirtApicRegs)
2380 {
2381 /* Enable APIC-register virtualization. */
2382 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2384
2385 /* Enable virtual-interrupt delivery. */
2386 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2387 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2388 }
2389#endif
2390
2391 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2392 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2393 * done dynamically. */
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2395 {
2396 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2397 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2399 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2400 AssertRCReturn(rc, rc);
2401 }
2402
2403 /* Enable RDTSCP. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2405 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2406
2407 /* Enable Pause-Loop exiting. */
2408 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2409 && pVM->hm.s.vmx.cPleGapTicks
2410 && pVM->hm.s.vmx.cPleWindowTicks)
2411 {
2412 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2413
2414 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2415 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2416 AssertRCReturn(rc, rc);
2417 }
2418
2419 if ((fVal & fZap) != fVal)
2420 {
2421 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2422 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2423 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2424 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2425 }
2426
2427 /* Commit it to the VMCS and update our cache. */
2428 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2429 AssertRCReturn(rc, rc);
2430 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2431
2432 return VINF_SUCCESS;
2433}
2434
2435
2436/**
2437 * Sets up processor-based VM-execution controls in the VMCS.
2438 *
2439 * @returns VBox status code.
2440 * @param pVCpu The cross context virtual CPU structure.
2441 *
2442 * @remarks We don't really care about optimizing vmwrites here as it's done only
2443 * once per VM and hence we don't care about VMCS-field cache comparisons.
2444 */
2445static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2446{
2447 PVM pVM = pVCpu->CTX_SUFF(pVM);
2448 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2449 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2452 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2453 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2454 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2455 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2456 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2458
2459 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2460 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2461 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2462 {
2463 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2464 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2465 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2466 }
2467
2468 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2469 if (!pVM->hm.s.fNestedPaging)
2470 {
2471 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2473 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2474 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2475 }
2476
2477 /* Use TPR shadowing if supported by the CPU. */
2478 if ( PDMHasApic(pVM)
2479 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480 {
2481 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2484 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2485 AssertRCReturn(rc, rc);
2486
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2488 /* CR8 writes cause a VM-exit based on TPR threshold. */
2489 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2490 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2491 }
2492 else
2493 {
2494 /*
2495 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2496 * Set this control only for 64-bit guests.
2497 */
2498 if (pVM->hm.s.fAllow64BitGuests)
2499 {
2500 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2501 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2502 }
2503 }
2504
2505 /* Use MSR-bitmaps if supported by the CPU. */
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2507 {
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2509
2510 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2512 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2513 AssertRCReturn(rc, rc);
2514
2515 /*
2516 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2517 * automatically using dedicated fields in the VMCS.
2518 */
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524#if HC_ARCH_BITS == 64
2525 /*
2526 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2527 */
2528 if (pVM->hm.s.fAllow64BitGuests)
2529 {
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534 }
2535#endif
2536 /*
2537 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2538 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2539 */
2540 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2541 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542
2543 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2544 }
2545
2546 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2547 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2549
2550 if ((fVal & fZap) != fVal)
2551 {
2552 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2553 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Commit it to the VMCS and update our cache. */
2559 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2560 AssertRCReturn(rc, rc);
2561 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2562
2563 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2564 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2565 return hmR0VmxSetupProcCtls2(pVCpu);
2566
2567 /* Sanity check, should not really happen. */
2568 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569 {
2570 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2571 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573 }
2574
2575 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2576 return VINF_SUCCESS;
2577}
2578
2579
2580/**
2581 * Sets up miscellaneous (everything other than Pin & Processor-based
2582 * VM-execution) control fields in the VMCS.
2583 *
2584 * @returns VBox status code.
2585 * @param pVCpu The cross context virtual CPU structure.
2586 */
2587static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2588{
2589 AssertPtr(pVCpu);
2590
2591 int rc = VERR_GENERAL_FAILURE;
2592
2593 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594#if 0
2595 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2596 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2598
2599 /*
2600 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2601 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2602 * We thus use the exception bitmap to control it rather than use both.
2603 */
2604 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2606
2607 /* All IO & IOIO instructions cause VM-exits. */
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2609 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2610
2611 /* Initialize the MSR-bitmap area. */
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2614 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 /* Setup MSR auto-load/store area. */
2619 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623 AssertRCReturn(rc, rc);
2624
2625 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2626 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2628 AssertRCReturn(rc, rc);
2629
2630 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2632 AssertRCReturn(rc, rc);
2633
2634 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2635#if 0
2636 /* Setup debug controls */
2637 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2638 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2639 AssertRCReturn(rc, rc);
2640#endif
2641
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2648 *
2649 * We shall setup those exception intercepts that don't change during the
2650 * lifetime of the VM here. The rest are done dynamically while loading the
2651 * guest state.
2652 *
2653 * @returns VBox status code.
2654 * @param pVCpu The cross context virtual CPU structure.
2655 */
2656static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2657{
2658 AssertPtr(pVCpu);
2659
2660 uint32_t uXcptBitmap;
2661
2662 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2663 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2664
2665 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2666 and writes, and because recursive #DBs can cause the CPU hang, we must always
2667 intercept #DB. */
2668 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2669
2670 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2671 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2672 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2673
2674 /* Commit it to the VMCS. */
2675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2676 AssertRCReturn(rc, rc);
2677
2678 /* Update our cache of the exception bitmap. */
2679 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2740 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2741 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2742 */
2743 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2744 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2745 || !pVM->hm.s.vmx.pRealModeTSS))
2746 {
2747 LogRelFunc(("Invalid real-on-v86 state.\n"));
2748 return VERR_INTERNAL_ERROR;
2749 }
2750
2751 /* Initialize these always, see hmR3InitFinalizeR0().*/
2752 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2753 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2754
2755 /* Setup the tagged-TLB flush handlers. */
2756 int rc = hmR0VmxSetupTaggedTlb(pVM);
2757 if (RT_FAILURE(rc))
2758 {
2759 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2760 return rc;
2761 }
2762
2763 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2764 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2765#if HC_ARCH_BITS == 64
2766 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2768 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2769 {
2770 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2771 }
2772#endif
2773
2774 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2775 RTCCUINTREG uHostCR4 = ASMGetCR4();
2776 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2777 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2778
2779 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2780 {
2781 PVMCPU pVCpu = &pVM->aCpus[i];
2782 AssertPtr(pVCpu);
2783 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2784
2785 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2786 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2787
2788 /* Set revision dword at the beginning of the VMCS structure. */
2789 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2790
2791 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2792 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 /* Load this VMCS as the current VMCS. */
2797 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2799 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801 rc = hmR0VmxSetupPinCtls(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804
2805 rc = hmR0VmxSetupProcCtls(pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2807 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808
2809 rc = hmR0VmxSetupMiscCtls(pVCpu);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 rc = hmR0VmxInitXcptBitmap(pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2816
2817#if HC_ARCH_BITS == 32
2818 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821#endif
2822
2823 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2824 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2826 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2827
2828 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2829
2830 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2831 }
2832
2833 return VINF_SUCCESS;
2834}
2835
2836
2837/**
2838 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2839 * the VMCS.
2840 *
2841 * @returns VBox status code.
2842 */
2843static int hmR0VmxExportHostControlRegs(void)
2844{
2845 RTCCUINTREG uReg = ASMGetCR0();
2846 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847 AssertRCReturn(rc, rc);
2848
2849 uReg = ASMGetCR3();
2850 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2851 AssertRCReturn(rc, rc);
2852
2853 uReg = ASMGetCR4();
2854 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2855 AssertRCReturn(rc, rc);
2856 return rc;
2857}
2858
2859
2860/**
2861 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2862 * the host-state area in the VMCS.
2863 *
2864 * @returns VBox status code.
2865 * @param pVCpu The cross context virtual CPU structure.
2866 */
2867static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2868{
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxExportHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890
2891 /*
2892 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893 * should -not- save the messed up state without restoring the original host-state,
2894 * see @bugref{7240}.
2895 *
2896 * This apparently can happen (most likely the FPU changes), deal with it rather than
2897 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2898 */
2899 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2900 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2901 {
2902 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2903 pVCpu->idCpu));
2904 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2905 }
2906 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2907#else
2908 RT_NOREF(pVCpu);
2909#endif
2910
2911 /*
2912 * Host DS, ES, FS and GS segment registers.
2913 */
2914#if HC_ARCH_BITS == 64
2915 RTSEL uSelDS = ASMGetDS();
2916 RTSEL uSelES = ASMGetES();
2917 RTSEL uSelFS = ASMGetFS();
2918 RTSEL uSelGS = ASMGetGS();
2919#else
2920 RTSEL uSelDS = 0;
2921 RTSEL uSelES = 0;
2922 RTSEL uSelFS = 0;
2923 RTSEL uSelGS = 0;
2924#endif
2925
2926 /*
2927 * Host CS and SS segment registers.
2928 */
2929 RTSEL uSelCS = ASMGetCS();
2930 RTSEL uSelSS = ASMGetSS();
2931
2932 /*
2933 * Host TR segment register.
2934 */
2935 RTSEL uSelTR = ASMGetTR();
2936
2937#if HC_ARCH_BITS == 64
2938 /*
2939 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2940 * gain VM-entry and restore them before we get preempted.
2941 *
2942 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2943 */
2944 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2945 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2946 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2947 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2948# undef VMXLOCAL_ADJUST_HOST_SEG
2949#endif
2950
2951 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2952 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2953 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2954 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2955 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2956 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2957 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2958 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2959 Assert(uSelCS);
2960 Assert(uSelTR);
2961
2962 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2963#if 0
2964 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2965 Assert(uSelSS != 0);
2966#endif
2967
2968 /* Write these host selector fields into the host-state area in the VMCS. */
2969 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2970 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2971#if HC_ARCH_BITS == 64
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2975 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2976#else
2977 NOREF(uSelDS);
2978 NOREF(uSelES);
2979 NOREF(uSelFS);
2980 NOREF(uSelGS);
2981#endif
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2983 AssertRCReturn(rc, rc);
2984
2985 /*
2986 * Host GDTR and IDTR.
2987 */
2988 RTGDTR Gdtr;
2989 RTIDTR Idtr;
2990 RT_ZERO(Gdtr);
2991 RT_ZERO(Idtr);
2992 ASMGetGDTR(&Gdtr);
2993 ASMGetIDTR(&Idtr);
2994 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2995 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2996 AssertRCReturn(rc, rc);
2997
2998#if HC_ARCH_BITS == 64
2999 /*
3000 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3001 * them to the maximum limit (0xffff) on every VM-exit.
3002 */
3003 if (Gdtr.cbGdt != 0xffff)
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3005
3006 /*
3007 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3008 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3009 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3010 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3011 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3012 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3013 * at 0xffff on hosts where we are sure it won't cause trouble.
3014 */
3015# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3016 if (Idtr.cbIdt < 0x0fff)
3017# else
3018 if (Idtr.cbIdt != 0xffff)
3019# endif
3020 {
3021 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3022 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024 }
3025#endif
3026
3027 /*
3028 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3029 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3030 * RPL should be too in most cases.
3031 */
3032 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3033 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3034
3035 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3036#if HC_ARCH_BITS == 64
3037 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3038
3039 /*
3040 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3041 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3042 * restoration if the host has something else. Task switching is not supported in 64-bit
3043 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3044 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3045 *
3046 * [1] See Intel spec. 3.5 "System Descriptor Types".
3047 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3048 */
3049 PVM pVM = pVCpu->CTX_SUFF(pVM);
3050 Assert(pDesc->System.u4Type == 11);
3051 if ( pDesc->System.u16LimitLow != 0x67
3052 || pDesc->System.u4LimitHigh)
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3055 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3056 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3057 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3058 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3059 }
3060
3061 /*
3062 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3063 */
3064 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3065 {
3066 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3067 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3068 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3069 {
3070 /* The GDT is read-only but the writable GDT is available. */
3071 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3072 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3073 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3074 AssertRCReturn(rc, rc);
3075 }
3076 }
3077#else
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return VINF_SUCCESS;
3100}
3101
3102
3103/**
3104 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3105 * host-state area of the VMCS.
3106 *
3107 * Theses MSRs will be automatically restored on the host after every successful
3108 * VM-exit.
3109 *
3110 * @returns VBox status code.
3111 * @param pVCpu The cross context virtual CPU structure.
3112 *
3113 * @remarks No-long-jump zone!!!
3114 */
3115static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3116{
3117 AssertPtr(pVCpu);
3118 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3119
3120 /*
3121 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3122 * rather than swapping them on every VM-entry.
3123 */
3124 hmR0VmxLazySaveHostMsrs(pVCpu);
3125
3126 /*
3127 * Host Sysenter MSRs.
3128 */
3129 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3130#if HC_ARCH_BITS == 32
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3133#else
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3136#endif
3137 AssertRCReturn(rc, rc);
3138
3139 /*
3140 * Host EFER MSR.
3141 *
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3143 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3144 */
3145 PVM pVM = pVCpu->CTX_SUFF(pVM);
3146 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3147 {
3148 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3149 AssertRCReturn(rc, rc);
3150 }
3151
3152 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3153
3154 return VINF_SUCCESS;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3163 * hmR0VMxExportGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 *
3168 * @remarks Requires EFER, CR4.
3169 * @remarks No-long-jump zone!!!
3170 */
3171static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3172{
3173#ifdef HMVMX_ALWAYS_SWAP_EFER
3174 return true;
3175#endif
3176
3177 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3178#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3179 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3180 if (CPUMIsGuestInLongModeEx(pCtx))
3181 return false;
3182#endif
3183
3184 PVM pVM = pVCpu->CTX_SUFF(pVM);
3185 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3186 uint64_t const u64GuestEfer = pCtx->msrEFER;
3187
3188 /*
3189 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3190 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3191 */
3192 if ( CPUMIsGuestInLongModeEx(pCtx)
3193 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3194 {
3195 return true;
3196 }
3197
3198 /*
3199 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3200 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3201 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3202 */
3203 if ( (pCtx->cr4 & X86_CR4_PAE)
3204 && (pCtx->cr0 & X86_CR0_PG)
3205 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3206 {
3207 /* Assert that host is PAE capable. */
3208 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3209 return true;
3210 }
3211
3212 return false;
3213}
3214
3215
3216/**
3217 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3218 *
3219 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3220 * see Intel spec. 24.8.1 "VM-entry controls".
3221 *
3222 * @returns VBox status code.
3223 * @param pVCpu The cross context virtual CPU structure.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3229{
3230 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3231 {
3232 PVM pVM = pVCpu->CTX_SUFF(pVM);
3233 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3234 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3235
3236 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3237 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3238
3239 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3240 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3241 {
3242 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3243 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3244 }
3245 else
3246 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3247
3248 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3249 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3250 && hmR0VmxShouldSwapEferMsr(pVCpu))
3251 {
3252 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3253 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3254 }
3255
3256 /*
3257 * The following should -not- be set (since we're not in SMM mode):
3258 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3259 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3260 */
3261
3262 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3263 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3264
3265 if ((fVal & fZap) != fVal)
3266 {
3267 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3268 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3269 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3271 }
3272
3273 /* Commit it to the VMCS and update our cache. */
3274 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3275 {
3276 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3277 AssertRCReturn(rc, rc);
3278 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3279 }
3280
3281 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3282 }
3283 return VINF_SUCCESS;
3284}
3285
3286
3287/**
3288 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3289 *
3290 * @returns VBox status code.
3291 * @param pVCpu The cross context virtual CPU structure.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3296{
3297 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3298 {
3299 PVM pVM = pVCpu->CTX_SUFF(pVM);
3300 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3301 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3302
3303 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3304 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3305
3306 /*
3307 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3308 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3309 * hmR0VmxExportHostMsrs().
3310 */
3311#if HC_ARCH_BITS == 64
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314#else
3315 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3316 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3317 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3318 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3319 {
3320 /* The switcher returns to long mode, EFER is managed by the switcher. */
3321 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3322 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3323 }
3324 else
3325 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3326#endif
3327
3328 /* If the newer VMCS fields for managing EFER exists, use it. */
3329 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3330 && hmR0VmxShouldSwapEferMsr(pVCpu))
3331 {
3332 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3333 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3334 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3335 }
3336
3337 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3338 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3339
3340 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3341 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3342 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3343
3344 /* Enable saving of the VMX preemption timer value on VM-exit. */
3345 if ( pVM->hm.s.vmx.fUsePreemptTimer
3346 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3347 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3348
3349 if ((fVal & fZap) != fVal)
3350 {
3351 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3352 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3353 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3354 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355 }
3356
3357 /* Commit it to the VMCS and update our cache. */
3358 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3359 {
3360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3361 AssertRCReturn(rc, rc);
3362 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3363 }
3364
3365 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3366 }
3367 return VINF_SUCCESS;
3368}
3369
3370
3371/**
3372 * Sets the TPR threshold in the VMCS.
3373 *
3374 * @returns VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure.
3376 * @param u32TprThreshold The TPR threshold (task-priority class only).
3377 */
3378DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3379{
3380 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3381 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3382 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3383}
3384
3385
3386/**
3387 * Exports the guest APIC TPR state into the VMCS.
3388 *
3389 * @returns VBox status code.
3390 * @param pVCpu The cross context virtual CPU structure.
3391 *
3392 * @remarks No-long-jump zone!!!
3393 */
3394static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3395{
3396 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3397 {
3398 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3399
3400 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3401 && APICIsEnabled(pVCpu))
3402 {
3403 /*
3404 * Setup TPR shadowing.
3405 */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3418 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3419 * priority of the pending interrupt so we can deliver the interrupt. If there
3420 * are no interrupts pending, set threshold to 0 to not cause any
3421 * TPR-below-threshold VM-exits.
3422 */
3423 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3424 uint32_t u32TprThreshold = 0;
3425 if (fPendingIntr)
3426 {
3427 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3428 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3429 const uint8_t u8TprPriority = u8Tpr >> 4;
3430 if (u8PendingPriority <= u8TprPriority)
3431 u32TprThreshold = u8PendingPriority;
3432 }
3433
3434 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3435 AssertRCReturn(rc, rc);
3436 }
3437 }
3438 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3439 }
3440 return VINF_SUCCESS;
3441}
3442
3443
3444/**
3445 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3446 *
3447 * @returns Guest's interruptibility-state.
3448 * @param pVCpu The cross context virtual CPU structure.
3449 *
3450 * @remarks No-long-jump zone!!!
3451 */
3452static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3453{
3454 /*
3455 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3456 */
3457 uint32_t fIntrState = 0;
3458 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459 {
3460 /* If inhibition is active, RIP & RFLAGS should've been accessed
3461 (i.e. read previously from the VMCS or from ring-3). */
3462 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3463#ifdef VBOX_STRICT
3464 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3465 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3466#endif
3467 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pCtx->eflags.Bits.u1IF)
3470 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler
3478 * without executing guest code in VT-x, the flag's condition to be cleared is
3479 * met and thus the cleared state is correct.
3480 */
3481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3482 }
3483 }
3484
3485 /*
3486 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3487 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3488 * setting this would block host-NMIs and IRET will not clear the blocking.
3489 *
3490 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3491 */
3492 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3493 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3494 {
3495 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3496 }
3497
3498 return fIntrState;
3499}
3500
3501
3502/**
3503 * Exports the guest's interruptibility-state into the guest-state area in the
3504 * VMCS.
3505 *
3506 * @returns VBox status code.
3507 * @param pVCpu The cross context virtual CPU structure.
3508 * @param fIntrState The interruptibility-state to set.
3509 */
3510static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3511{
3512 NOREF(pVCpu);
3513 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3514 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3515 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3516}
3517
3518
3519/**
3520 * Exports the exception intercepts required for guest execution in the VMCS.
3521 *
3522 * @returns VBox status code.
3523 * @param pVCpu The cross context virtual CPU structure.
3524 *
3525 * @remarks No-long-jump zone!!!
3526 */
3527static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3528{
3529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3530 {
3531 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3532
3533 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3534 if (pVCpu->hm.s.fGIMTrapXcptUD)
3535 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3536#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3537 else
3538 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3539#endif
3540
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3542 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3543
3544 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3545 {
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3547 AssertRCReturn(rc, rc);
3548 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3549 }
3550
3551 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3552 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3553 }
3554 return VINF_SUCCESS;
3555}
3556
3557
3558/**
3559 * Exports the guest's RIP into the guest-state area in the VMCS.
3560 *
3561 * @returns VBox status code.
3562 * @param pVCpu The cross context virtual CPU structure.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3570 {
3571 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3572
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3574 AssertRCReturn(rc, rc);
3575
3576 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3577 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Exports the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 *
3589 * @remarks No-long-jump zone!!!
3590 */
3591static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3592{
3593 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3594 {
3595 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3596
3597 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3601 }
3602 return VINF_SUCCESS;
3603}
3604
3605
3606/**
3607 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 *
3612 * @remarks No-long-jump zone!!!
3613 */
3614static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3615{
3616 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617 {
3618 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3619
3620 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621 Let us assert it as such and use 32-bit VMWRITE. */
3622 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3623 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3624 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3625 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3626
3627 /*
3628 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3629 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3630 * can run the real-mode guest code under Virtual 8086 mode.
3631 */
3632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633 {
3634 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3637 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639 }
3640
3641 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3642 AssertRCReturn(rc, rc);
3643
3644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3645 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3646 }
3647 return VINF_SUCCESS;
3648}
3649
3650
3651/**
3652 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3653 *
3654 * The guest FPU state is always pre-loaded hence we don't need to bother about
3655 * sharing FPU related CR0 bits between the guest and host.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu The cross context virtual CPU structure.
3659 *
3660 * @remarks No-long-jump zone!!!
3661 */
3662static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3663{
3664 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3665 {
3666 PVM pVM = pVCpu->CTX_SUFF(pVM);
3667 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3668 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3669
3670 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3671 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3672
3673 /*
3674 * Setup VT-x's view of the guest CR0.
3675 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3676 */
3677 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3678 if (pVM->hm.s.fNestedPaging)
3679 {
3680 if (CPUMIsGuestPagingEnabled(pVCpu))
3681 {
3682 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3683 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3684 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3685 }
3686 else
3687 {
3688 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3689 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692
3693 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3694 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3695 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696 }
3697 else
3698 {
3699 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3700 u32GuestCr0 |= X86_CR0_WP;
3701 }
3702
3703 /*
3704 * Guest FPU bits.
3705 *
3706 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3707 * using CR0.TS.
3708 *
3709 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3710 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3711 */
3712 u32GuestCr0 |= X86_CR0_NE;
3713
3714 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3715 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3716
3717 /*
3718 * Update exception intercepts.
3719 */
3720 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3721 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3722 {
3723 Assert(PDMVmmDevHeapIsEnabled(pVM));
3724 Assert(pVM->hm.s.vmx.pRealModeTSS);
3725 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3726 }
3727 else
3728 {
3729 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3730 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3731 if (fInterceptMF)
3732 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3733 }
3734
3735 /* Additional intercepts for debugging, define these yourself explicitly. */
3736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3737 uXcptBitmap |= 0
3738 | RT_BIT(X86_XCPT_BP)
3739 | RT_BIT(X86_XCPT_DE)
3740 | RT_BIT(X86_XCPT_NM)
3741 | RT_BIT(X86_XCPT_TS)
3742 | RT_BIT(X86_XCPT_UD)
3743 | RT_BIT(X86_XCPT_NP)
3744 | RT_BIT(X86_XCPT_SS)
3745 | RT_BIT(X86_XCPT_GP)
3746 | RT_BIT(X86_XCPT_PF)
3747 | RT_BIT(X86_XCPT_MF)
3748 ;
3749#elif defined(HMVMX_ALWAYS_TRAP_PF)
3750 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3751#endif
3752 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3798 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3799 AssertRCReturn(rc, rc);
3800
3801 /* Update our caches. */
3802 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3803 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3804 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3805
3806 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3807
3808 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3809 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3810 }
3811
3812 return VINF_SUCCESS;
3813}
3814
3815
3816/**
3817 * Exports the guest control registers (CR3, CR4) into the guest-state area
3818 * in the VMCS.
3819 *
3820 * @returns VBox strict status code.
3821 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3822 * without unrestricted guest access and the VMMDev is not presently
3823 * mapped (e.g. EFI32).
3824 *
3825 * @param pVCpu The cross context virtual CPU structure.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3845
3846 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3847 if (pVM->hm.s.fNestedPaging)
3848 {
3849 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3850
3851 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3852 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3854 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3855
3856 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3857 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3858 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3859
3860 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3861 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3862 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3863 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3864 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3865 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3866 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867
3868 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3869 AssertRCReturn(rc, rc);
3870
3871 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3872 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3873 || CPUMIsGuestPagingEnabledEx(pCtx))
3874 {
3875 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3876 if (CPUMIsGuestInPAEModeEx(pCtx))
3877 {
3878 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3879 AssertRCReturn(rc, rc);
3880 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3883 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3884 AssertRCReturn(rc, rc);
3885 }
3886
3887 /*
3888 * The guest's view of its CR3 is unblemished with Nested Paging when the
3889 * guest is using paging or we have unrestricted guest execution to handle
3890 * the guest when it's not using paging.
3891 */
3892 GCPhysGuestCR3 = pCtx->cr3;
3893 }
3894 else
3895 {
3896 /*
3897 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3898 * thinks it accesses physical memory directly, we use our identity-mapped
3899 * page table to map guest-linear to guest-physical addresses. EPT takes care
3900 * of translating it to host-physical addresses.
3901 */
3902 RTGCPHYS GCPhys;
3903 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3904
3905 /* We obtain it here every time as the guest could have relocated this PCI region. */
3906 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907 if (RT_SUCCESS(rc))
3908 { /* likely */ }
3909 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3910 {
3911 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3912 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3913 }
3914 else
3915 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3916
3917 GCPhysGuestCR3 = GCPhys;
3918 }
3919
3920 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3921 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922 AssertRCReturn(rc, rc);
3923 }
3924 else
3925 {
3926 /* Non-nested paging case, just use the hypervisor's CR3. */
3927 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3928
3929 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3930 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3931 AssertRCReturn(rc, rc);
3932 }
3933
3934 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3935 }
3936
3937 /*
3938 * Guest CR4.
3939 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3940 */
3941 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3942 {
3943 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3944 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3945 Assert(!RT_HI_U32(pCtx->cr4));
3946
3947 uint32_t u32GuestCr4 = pCtx->cr4;
3948 uint32_t const u32ShadowCr4 = pCtx->cr4;
3949
3950 /*
3951 * Setup VT-x's view of the guest CR4.
3952 *
3953 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3954 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3955 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3956 *
3957 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958 */
3959 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960 {
3961 Assert(pVM->hm.s.vmx.pRealModeTSS);
3962 Assert(PDMVmmDevHeapIsEnabled(pVM));
3963 u32GuestCr4 &= ~X86_CR4_VME;
3964 }
3965
3966 if (pVM->hm.s.fNestedPaging)
3967 {
3968 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3969 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3970 {
3971 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972 u32GuestCr4 |= X86_CR4_PSE;
3973 /* Our identity mapping is a 32-bit page directory. */
3974 u32GuestCr4 &= ~X86_CR4_PAE;
3975 }
3976 /* else use guest CR4.*/
3977 }
3978 else
3979 {
3980 /*
3981 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983 */
3984 switch (pVCpu->hm.s.enmShadowMode)
3985 {
3986 case PGMMODE_REAL: /* Real-mode. */
3987 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988 case PGMMODE_32_BIT: /* 32-bit paging. */
3989 {
3990 u32GuestCr4 &= ~X86_CR4_PAE;
3991 break;
3992 }
3993
3994 case PGMMODE_PAE: /* PAE paging. */
3995 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996 {
3997 u32GuestCr4 |= X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004 break;
4005#endif
4006 default:
4007 AssertFailed();
4008 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009 }
4010 }
4011
4012 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 u32GuestCr4 |= fSetCr4;
4016 u32GuestCr4 &= fZapCr4;
4017
4018 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4019 that would cause a VM-exit. */
4020 uint32_t u32Cr4Mask = X86_CR4_VME
4021 | X86_CR4_PAE
4022 | X86_CR4_PGE
4023 | X86_CR4_PSE
4024 | X86_CR4_VMXE;
4025 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4026 u32Cr4Mask |= X86_CR4_OSXSAVE;
4027 if (pVM->cpum.ro.GuestFeatures.fPcid)
4028 u32Cr4Mask |= X86_CR4_PCIDE;
4029
4030 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4031 into the VMCS and update our cache. */
4032 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4034 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4035 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4036 AssertRCReturn(rc, rc);
4037 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4038
4039 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4040 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4041
4042 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4043
4044 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4045 fZapCr4));
4046 }
4047 return rc;
4048}
4049
4050
4051/**
4052 * Exports the guest debug registers into the guest-state area in the VMCS.
4053 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4054 *
4055 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4056 *
4057 * @returns VBox status code.
4058 * @param pVCpu The cross context virtual CPU structure.
4059 *
4060 * @remarks No-long-jump zone!!!
4061 */
4062static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4063{
4064 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4068 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4069 {
4070 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4072 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4073 }
4074#endif
4075
4076 bool fSteppingDB = false;
4077 bool fInterceptMovDRx = false;
4078 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4079 if (pVCpu->hm.s.fSingleInstruction)
4080 {
4081 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4082 PVM pVM = pVCpu->CTX_SUFF(pVM);
4083 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4084 {
4085 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4086 Assert(fSteppingDB == false);
4087 }
4088 else
4089 {
4090 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4091 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4092 pVCpu->hm.s.fClearTrapFlag = true;
4093 fSteppingDB = true;
4094 }
4095 }
4096
4097 uint32_t u32GuestDr7;
4098 if ( fSteppingDB
4099 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4100 {
4101 /*
4102 * Use the combined guest and host DRx values found in the hypervisor register set
4103 * because the debugger has breakpoints active or someone is single stepping on the
4104 * host side without a monitor trap flag.
4105 *
4106 * Note! DBGF expects a clean DR6 state before executing guest code.
4107 */
4108#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4110 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4111 {
4112 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4113 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4114 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4115 }
4116 else
4117#endif
4118 if (!CPUMIsHyperDebugStateActive(pVCpu))
4119 {
4120 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4121 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4122 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4123 }
4124
4125 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4126 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4127 pVCpu->hm.s.fUsingHyperDR7 = true;
4128 fInterceptMovDRx = true;
4129 }
4130 else
4131 {
4132 /*
4133 * If the guest has enabled debug registers, we need to load them prior to
4134 * executing guest code so they'll trigger at the right time.
4135 */
4136 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4137 {
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4140 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146 }
4147 else
4148#endif
4149 if (!CPUMIsGuestDebugStateActive(pVCpu))
4150 {
4151 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155 }
4156 Assert(!fInterceptMovDRx);
4157 }
4158 /*
4159 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160 * must intercept #DB in order to maintain a correct DR6 guest value, and
4161 * because we need to intercept it to prevent nested #DBs from hanging the
4162 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4163 */
4164#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4165 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4166 && !CPUMIsGuestDebugStateActive(pVCpu))
4167#else
4168 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4169#endif
4170 {
4171 fInterceptMovDRx = true;
4172 }
4173
4174 /* Update DR7 with the actual guest value. */
4175 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4176 pVCpu->hm.s.fUsingHyperDR7 = false;
4177 }
4178
4179 if (fInterceptMovDRx)
4180 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181 else
4182 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4183
4184 /*
4185 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4186 * monitor-trap flag and update our cache.
4187 */
4188 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4189 {
4190 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4191 AssertRCReturn(rc2, rc2);
4192 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4193 }
4194
4195 /*
4196 * Update guest DR7.
4197 */
4198 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4199 AssertRCReturn(rc, rc);
4200
4201 return VINF_SUCCESS;
4202}
4203
4204
4205#ifdef VBOX_STRICT
4206/**
4207 * Strict function to validate segment registers.
4208 *
4209 * @param pVCpu The cross context virtual CPU structure.
4210 *
4211 * @remarks Will import guest CR0 on strict builds during validation of
4212 * segments.
4213 */
4214static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4215{
4216 /*
4217 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4218 *
4219 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4220 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4221 * and doesn't change the guest-context value.
4222 */
4223 PVM pVM = pVCpu->CTX_SUFF(pVM);
4224 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4225 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4226 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4227 && ( !CPUMIsGuestInRealModeEx(pCtx)
4228 && !CPUMIsGuestInV86ModeEx(pCtx)))
4229 {
4230 /* Protected mode checks */
4231 /* CS */
4232 Assert(pCtx->cs.Attr.n.u1Present);
4233 Assert(!(pCtx->cs.Attr.u & 0xf00));
4234 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4235 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4236 || !(pCtx->cs.Attr.n.u1Granularity));
4237 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4238 || (pCtx->cs.Attr.n.u1Granularity));
4239 /* CS cannot be loaded with NULL in protected mode. */
4240 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4241 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4242 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4243 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4244 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4245 else
4246 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4247 /* SS */
4248 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4249 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4250 if ( !(pCtx->cr0 & X86_CR0_PE)
4251 || pCtx->cs.Attr.n.u4Type == 3)
4252 {
4253 Assert(!pCtx->ss.Attr.n.u2Dpl);
4254 }
4255 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4256 {
4257 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4258 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4259 Assert(pCtx->ss.Attr.n.u1Present);
4260 Assert(!(pCtx->ss.Attr.u & 0xf00));
4261 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->ss.Attr.n.u1Granularity));
4264 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4265 || (pCtx->ss.Attr.n.u1Granularity));
4266 }
4267 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4268 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4269 {
4270 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4271 Assert(pCtx->ds.Attr.n.u1Present);
4272 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4273 Assert(!(pCtx->ds.Attr.u & 0xf00));
4274 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4275 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4276 || !(pCtx->ds.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4278 || (pCtx->ds.Attr.n.u1Granularity));
4279 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4280 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4281 }
4282 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4285 Assert(pCtx->es.Attr.n.u1Present);
4286 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4287 Assert(!(pCtx->es.Attr.u & 0xf00));
4288 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->es.Attr.n.u1Granularity));
4291 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4292 || (pCtx->es.Attr.n.u1Granularity));
4293 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4294 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4295 }
4296 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4297 {
4298 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4299 Assert(pCtx->fs.Attr.n.u1Present);
4300 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4301 Assert(!(pCtx->fs.Attr.u & 0xf00));
4302 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4303 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4304 || !(pCtx->fs.Attr.n.u1Granularity));
4305 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4306 || (pCtx->fs.Attr.n.u1Granularity));
4307 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4308 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4309 }
4310 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4311 {
4312 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313 Assert(pCtx->gs.Attr.n.u1Present);
4314 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4315 Assert(!(pCtx->gs.Attr.u & 0xf00));
4316 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4317 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4318 || !(pCtx->gs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4320 || (pCtx->gs.Attr.n.u1Granularity));
4321 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323 }
4324 /* 64-bit capable CPUs. */
4325# if HC_ARCH_BITS == 64
4326 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4327 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4328 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4329 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4330# endif
4331 }
4332 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4333 || ( CPUMIsGuestInRealModeEx(pCtx)
4334 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4335 {
4336 /* Real and v86 mode checks. */
4337 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4338 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4339 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4340 {
4341 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4342 }
4343 else
4344 {
4345 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4346 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4347 }
4348
4349 /* CS */
4350 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4351 Assert(pCtx->cs.u32Limit == 0xffff);
4352 Assert(u32CSAttr == 0xf3);
4353 /* SS */
4354 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4355 Assert(pCtx->ss.u32Limit == 0xffff);
4356 Assert(u32SSAttr == 0xf3);
4357 /* DS */
4358 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4359 Assert(pCtx->ds.u32Limit == 0xffff);
4360 Assert(u32DSAttr == 0xf3);
4361 /* ES */
4362 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4363 Assert(pCtx->es.u32Limit == 0xffff);
4364 Assert(u32ESAttr == 0xf3);
4365 /* FS */
4366 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4367 Assert(pCtx->fs.u32Limit == 0xffff);
4368 Assert(u32FSAttr == 0xf3);
4369 /* GS */
4370 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4371 Assert(pCtx->gs.u32Limit == 0xffff);
4372 Assert(u32GSAttr == 0xf3);
4373 /* 64-bit capable CPUs. */
4374# if HC_ARCH_BITS == 64
4375 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4376 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4377 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4378 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4379# endif
4380 }
4381}
4382#endif /* VBOX_STRICT */
4383
4384
4385/**
4386 * Exports a guest segment register into the guest-state area in the VMCS.
4387 *
4388 * @returns VBox status code.
4389 * @param pVCpu The cross context virtual CPU structure.
4390 * @param idxSel Index of the selector in the VMCS.
4391 * @param idxLimit Index of the segment limit in the VMCS.
4392 * @param idxBase Index of the segment base in the VMCS.
4393 * @param idxAccess Index of the access rights of the segment in the VMCS.
4394 * @param pSelReg Pointer to the segment selector.
4395 *
4396 * @remarks No-long-jump zone!!!
4397 */
4398static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4399 PCCPUMSELREG pSelReg)
4400{
4401 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4402 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4403 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4404 AssertRCReturn(rc, rc);
4405
4406 uint32_t u32Access = pSelReg->Attr.u;
4407 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4408 {
4409 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4410 u32Access = 0xf3;
4411 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4412 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4413 }
4414 else
4415 {
4416 /*
4417 * The way to differentiate between whether this is really a null selector or was just
4418 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4419 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4420 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4421 * NULL selectors loaded in protected-mode have their attribute as 0.
4422 */
4423 if (!u32Access)
4424 u32Access = X86DESCATTR_UNUSABLE;
4425 }
4426
4427 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4428 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4429 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4430
4431 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4432 AssertRCReturn(rc, rc);
4433 return rc;
4434}
4435
4436
4437/**
4438 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4439 * into the guest-state area in the VMCS.
4440 *
4441 * @returns VBox status code.
4442 * @param pVCpu The cross context virtual CPU structure.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4453
4454 /*
4455 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4456 */
4457 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4458 {
4459#ifdef VBOX_WITH_REM
4460 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4461 {
4462 Assert(pVM->hm.s.vmx.pRealModeTSS);
4463 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4464 if ( pVCpu->hm.s.vmx.fWasInRealMode
4465 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4466 {
4467 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4468 in real-mode (e.g. OpenBSD 4.0) */
4469 REMFlushTBs(pVM);
4470 Log4Func(("Switch to protected mode detected!\n"));
4471 pVCpu->hm.s.vmx.fWasInRealMode = false;
4472 }
4473 }
4474#endif
4475 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4476 {
4477 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4478 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4480 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4481 AssertRCReturn(rc, rc);
4482 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4483 }
4484
4485 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4486 {
4487 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4488 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4490 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4491 AssertRCReturn(rc, rc);
4492 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4493 }
4494
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4496 {
4497 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4500 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4501 AssertRCReturn(rc, rc);
4502 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4503 }
4504
4505 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4506 {
4507 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4508 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4509 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4510 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4511 AssertRCReturn(rc, rc);
4512 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4513 }
4514
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4516 {
4517 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4519 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4520 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4521 AssertRCReturn(rc, rc);
4522 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4523 }
4524
4525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4526 {
4527 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4528 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4530 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4531 AssertRCReturn(rc, rc);
4532 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4533 }
4534
4535#ifdef VBOX_STRICT
4536 hmR0VmxValidateSegmentRegs(pVCpu);
4537#endif
4538
4539 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4540 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4541 }
4542
4543 /*
4544 * Guest TR.
4545 */
4546 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4547 {
4548 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4549
4550 /*
4551 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4552 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4553 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4554 */
4555 uint16_t u16Sel = 0;
4556 uint32_t u32Limit = 0;
4557 uint64_t u64Base = 0;
4558 uint32_t u32AccessRights = 0;
4559
4560 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4561 {
4562 u16Sel = pCtx->tr.Sel;
4563 u32Limit = pCtx->tr.u32Limit;
4564 u64Base = pCtx->tr.u64Base;
4565 u32AccessRights = pCtx->tr.Attr.u;
4566 }
4567 else
4568 {
4569 Assert(pVM->hm.s.vmx.pRealModeTSS);
4570 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4571
4572 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4573 RTGCPHYS GCPhys;
4574 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4575 AssertRCReturn(rc, rc);
4576
4577 X86DESCATTR DescAttr;
4578 DescAttr.u = 0;
4579 DescAttr.n.u1Present = 1;
4580 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4581
4582 u16Sel = 0;
4583 u32Limit = HM_VTX_TSS_SIZE;
4584 u64Base = GCPhys; /* in real-mode phys = virt. */
4585 u32AccessRights = DescAttr.u;
4586 }
4587
4588 /* Validate. */
4589 Assert(!(u16Sel & RT_BIT(2)));
4590 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4591 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4592 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4593 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4594 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4595 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4596 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4597 Assert( (u32Limit & 0xfff) == 0xfff
4598 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4599 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4600 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4601
4602 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4603 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4604 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4605 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4606 AssertRCReturn(rc, rc);
4607
4608 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4609 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4610 }
4611
4612 /*
4613 * Guest GDTR.
4614 */
4615 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4616 {
4617 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4618
4619 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4620 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4621 AssertRCReturn(rc, rc);
4622
4623 /* Validate. */
4624 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4625
4626 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4627 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4628 }
4629
4630 /*
4631 * Guest LDTR.
4632 */
4633 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4634 {
4635 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4636
4637 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4638 uint32_t u32Access = 0;
4639 if (!pCtx->ldtr.Attr.u)
4640 u32Access = X86DESCATTR_UNUSABLE;
4641 else
4642 u32Access = pCtx->ldtr.Attr.u;
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4645 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4647 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4648 AssertRCReturn(rc, rc);
4649
4650 /* Validate. */
4651 if (!(u32Access & X86DESCATTR_UNUSABLE))
4652 {
4653 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4654 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4655 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4656 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4657 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4658 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4659 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4660 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4661 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4662 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4663 }
4664
4665 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4666 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4667 }
4668
4669 /*
4670 * Guest IDTR.
4671 */
4672 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4673 {
4674 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4675
4676 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4677 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4678 AssertRCReturn(rc, rc);
4679
4680 /* Validate. */
4681 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4682
4683 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4684 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4685 }
4686
4687 return VINF_SUCCESS;
4688}
4689
4690
4691/**
4692 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4693 * areas.
4694 *
4695 * These MSRs will automatically be loaded to the host CPU on every successful
4696 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4697 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4698 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4699 *
4700 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4701 *
4702 * @returns VBox status code.
4703 * @param pVCpu The cross context virtual CPU structure.
4704 *
4705 * @remarks No-long-jump zone!!!
4706 */
4707static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4708{
4709 AssertPtr(pVCpu);
4710 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4711
4712 /*
4713 * MSRs that we use the auto-load/store MSR area in the VMCS.
4714 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4715 */
4716 PVM pVM = pVCpu->CTX_SUFF(pVM);
4717 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4718 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4719 {
4720 if (pVM->hm.s.fAllow64BitGuests)
4721 {
4722#if HC_ARCH_BITS == 32
4723 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4724
4725 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4727 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4728 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4729 AssertRCReturn(rc, rc);
4730# ifdef LOG_ENABLED
4731 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4732 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4733 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4734# endif
4735#endif
4736 }
4737 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4738 }
4739
4740 /*
4741 * Guest Sysenter MSRs.
4742 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4743 * VM-exits on WRMSRs for these MSRs.
4744 */
4745 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4746 {
4747 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4748
4749 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4750 {
4751 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4752 AssertRCReturn(rc, rc);
4753 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4754 }
4755
4756 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4757 {
4758 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4759 AssertRCReturn(rc, rc);
4760 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4761 }
4762
4763 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4764 {
4765 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4766 AssertRCReturn(rc, rc);
4767 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4768 }
4769 }
4770
4771 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4772 {
4773 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4774
4775 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4776 {
4777 /*
4778 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4779 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4780 */
4781 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4782 {
4783 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4784 AssertRCReturn(rc,rc);
4785 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4786 }
4787 else
4788 {
4789 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4790 NULL /* pfAddedAndUpdated */);
4791 AssertRCReturn(rc, rc);
4792
4793 /* We need to intercept reads too, see @bugref{7386#c16}. */
4794 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4795 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4796 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4797 pVCpu->hm.s.vmx.cMsrs));
4798 }
4799 }
4800 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4801 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4802 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4803 }
4804
4805 return VINF_SUCCESS;
4806}
4807
4808
4809#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4810/**
4811 * Check if guest state allows safe use of 32-bit switcher again.
4812 *
4813 * Segment bases and protected mode structures must be 32-bit addressable
4814 * because the 32-bit switcher will ignore high dword when writing these VMCS
4815 * fields. See @bugref{8432} for details.
4816 *
4817 * @returns true if safe, false if must continue to use the 64-bit switcher.
4818 * @param pCtx Pointer to the guest-CPU context.
4819 *
4820 * @remarks No-long-jump zone!!!
4821 */
4822static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4823{
4824 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834
4835 /* All good, bases are 32-bit. */
4836 return true;
4837}
4838#endif
4839
4840
4841/**
4842 * Selects up the appropriate function to run guest code.
4843 *
4844 * @returns VBox status code.
4845 * @param pVCpu The cross context virtual CPU structure.
4846 *
4847 * @remarks No-long-jump zone!!!
4848 */
4849static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4850{
4851 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4852 if (CPUMIsGuestInLongModeEx(pCtx))
4853 {
4854#ifndef VBOX_ENABLE_64_BITS_GUESTS
4855 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4856#endif
4857 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4858#if HC_ARCH_BITS == 32
4859 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4860 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4861 {
4862#ifdef VBOX_STRICT
4863 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4864 {
4865 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4866 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4867 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4868 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4869 | HM_CHANGED_VMX_ENTRY_CTLS
4870 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4871 }
4872#endif
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4874
4875 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4876 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4877 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4878 Log4Func(("Selected 64-bit switcher\n"));
4879 }
4880#else
4881 /* 64-bit host. */
4882 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4883#endif
4884 }
4885 else
4886 {
4887 /* Guest is not in long mode, use the 32-bit handler. */
4888#if HC_ARCH_BITS == 32
4889 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4890 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4891 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4892 {
4893# ifdef VBOX_STRICT
4894 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4895 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4896 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4897 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4898 | HM_CHANGED_VMX_ENTRY_CTLS
4899 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4900# endif
4901 }
4902# ifdef VBOX_ENABLE_64_BITS_GUESTS
4903 /*
4904 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4905 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4906 * switcher flag because now we know the guest is in a sane state where it's safe
4907 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4908 * the much faster 32-bit switcher again.
4909 */
4910 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4911 {
4912 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4913 Log4Func(("Selected 32-bit switcher\n"));
4914 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4915 }
4916 else
4917 {
4918 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4919 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4920 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4921 {
4922 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4923 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4924 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4925 | HM_CHANGED_VMX_ENTRY_CTLS
4926 | HM_CHANGED_VMX_EXIT_CTLS
4927 | HM_CHANGED_HOST_CONTEXT);
4928 Log4Func(("Selected 32-bit switcher (safe)\n"));
4929 }
4930 }
4931# else
4932 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4933# endif
4934#else
4935 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4936#endif
4937 }
4938 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4939 return VINF_SUCCESS;
4940}
4941
4942
4943/**
4944 * Wrapper for running the guest code in VT-x.
4945 *
4946 * @returns VBox status code, no informational status codes.
4947 * @param pVCpu The cross context virtual CPU structure.
4948 *
4949 * @remarks No-long-jump zone!!!
4950 */
4951DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4952{
4953 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4954 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4955 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4956
4957 /*
4958 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4959 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4960 * callee-saved and thus the need for this XMM wrapper.
4961 *
4962 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4963 */
4964 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4965 /** @todo Add stats for resume vs launch. */
4966 PVM pVM = pVCpu->CTX_SUFF(pVM);
4967#ifdef VBOX_WITH_KERNEL_USING_XMM
4968 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4969#else
4970 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4971#endif
4972 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4973 return rc;
4974}
4975
4976
4977/**
4978 * Reports world-switch error and dumps some useful debug info.
4979 *
4980 * @param pVCpu The cross context virtual CPU structure.
4981 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4982 * @param pVmxTransient Pointer to the VMX transient structure (only
4983 * exitReason updated).
4984 */
4985static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4986{
4987 Assert(pVCpu);
4988 Assert(pVmxTransient);
4989 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4990
4991 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4992 switch (rcVMRun)
4993 {
4994 case VERR_VMX_INVALID_VMXON_PTR:
4995 AssertFailed();
4996 break;
4997 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4998 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4999 {
5000 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5001 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5002 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5003 AssertRC(rc);
5004
5005 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5006 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5007 Cannot do it here as we may have been long preempted. */
5008
5009#ifdef VBOX_STRICT
5010 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5011 pVmxTransient->uExitReason));
5012 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5013 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5014 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5015 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5016 else
5017 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5018 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5019 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5020
5021 /* VMX control bits. */
5022 uint32_t u32Val;
5023 uint64_t u64Val;
5024 RTHCUINTREG uHCReg;
5025 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5026 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5027 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5028 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5029 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5030 {
5031 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5032 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5033 }
5034 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5035 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5060 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5061 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5062 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5063 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5065 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5066 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5067 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5068 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5069 {
5070 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5071 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5072 }
5073
5074 /* Guest bits. */
5075 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5076 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5077 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5078 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5079 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5080 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5081 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5082 {
5083 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5085 }
5086
5087 /* Host bits. */
5088 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5089 Log4(("Host CR0 %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5091 Log4(("Host CR3 %#RHr\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5093 Log4(("Host CR4 %#RHr\n", uHCReg));
5094
5095 RTGDTR HostGdtr;
5096 PCX86DESCHC pDesc;
5097 ASMGetGDTR(&HostGdtr);
5098 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5099 Log4(("Host CS %#08x\n", u32Val));
5100 if (u32Val < HostGdtr.cbGdt)
5101 {
5102 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5103 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5104 }
5105
5106 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5107 Log4(("Host DS %#08x\n", u32Val));
5108 if (u32Val < HostGdtr.cbGdt)
5109 {
5110 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5111 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5112 }
5113
5114 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5115 Log4(("Host ES %#08x\n", u32Val));
5116 if (u32Val < HostGdtr.cbGdt)
5117 {
5118 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5119 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5120 }
5121
5122 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5123 Log4(("Host FS %#08x\n", u32Val));
5124 if (u32Val < HostGdtr.cbGdt)
5125 {
5126 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5127 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5128 }
5129
5130 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5131 Log4(("Host GS %#08x\n", u32Val));
5132 if (u32Val < HostGdtr.cbGdt)
5133 {
5134 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5135 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5136 }
5137
5138 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5139 Log4(("Host SS %#08x\n", u32Val));
5140 if (u32Val < HostGdtr.cbGdt)
5141 {
5142 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5143 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5144 }
5145
5146 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5147 Log4(("Host TR %#08x\n", u32Val));
5148 if (u32Val < HostGdtr.cbGdt)
5149 {
5150 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5151 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5152 }
5153
5154 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5155 Log4(("Host TR Base %#RHv\n", uHCReg));
5156 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5157 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5158 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5159 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5160 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5161 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5163 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5164 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5165 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5167 Log4(("Host RSP %#RHv\n", uHCReg));
5168 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5169 Log4(("Host RIP %#RHv\n", uHCReg));
5170# if HC_ARCH_BITS == 64
5171 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5172 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5173 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5174 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5175 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5176 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5177# endif
5178#endif /* VBOX_STRICT */
5179 break;
5180 }
5181
5182 default:
5183 /* Impossible */
5184 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5185 break;
5186 }
5187}
5188
5189
5190#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5191#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5192# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5193#endif
5194#ifdef VBOX_STRICT
5195static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5196{
5197 switch (idxField)
5198 {
5199 case VMX_VMCS_GUEST_RIP:
5200 case VMX_VMCS_GUEST_RSP:
5201 case VMX_VMCS_GUEST_SYSENTER_EIP:
5202 case VMX_VMCS_GUEST_SYSENTER_ESP:
5203 case VMX_VMCS_GUEST_GDTR_BASE:
5204 case VMX_VMCS_GUEST_IDTR_BASE:
5205 case VMX_VMCS_GUEST_CS_BASE:
5206 case VMX_VMCS_GUEST_DS_BASE:
5207 case VMX_VMCS_GUEST_ES_BASE:
5208 case VMX_VMCS_GUEST_FS_BASE:
5209 case VMX_VMCS_GUEST_GS_BASE:
5210 case VMX_VMCS_GUEST_SS_BASE:
5211 case VMX_VMCS_GUEST_LDTR_BASE:
5212 case VMX_VMCS_GUEST_TR_BASE:
5213 case VMX_VMCS_GUEST_CR3:
5214 return true;
5215 }
5216 return false;
5217}
5218
5219static bool hmR0VmxIsValidReadField(uint32_t idxField)
5220{
5221 switch (idxField)
5222 {
5223 /* Read-only fields. */
5224 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5225 return true;
5226 }
5227 /* Remaining readable fields should also be writable. */
5228 return hmR0VmxIsValidWriteField(idxField);
5229}
5230#endif /* VBOX_STRICT */
5231
5232
5233/**
5234 * Executes the specified handler in 64-bit mode.
5235 *
5236 * @returns VBox status code (no informational status codes).
5237 * @param pVCpu The cross context virtual CPU structure.
5238 * @param enmOp The operation to perform.
5239 * @param cParams Number of parameters.
5240 * @param paParam Array of 32-bit parameters.
5241 */
5242VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5243{
5244 PVM pVM = pVCpu->CTX_SUFF(pVM);
5245 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5246 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5247 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5248 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5249
5250#ifdef VBOX_STRICT
5251 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5252 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5253
5254 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5255 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5256#endif
5257
5258 /* Disable interrupts. */
5259 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5260
5261#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5262 RTCPUID idHostCpu = RTMpCpuId();
5263 CPUMR0SetLApic(pVCpu, idHostCpu);
5264#endif
5265
5266 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5267 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5268
5269 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5270 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5271 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5272
5273 /* Leave VMX Root Mode. */
5274 VMXDisable();
5275
5276 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5277
5278 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5279 CPUMSetHyperEIP(pVCpu, enmOp);
5280 for (int i = (int)cParams - 1; i >= 0; i--)
5281 CPUMPushHyper(pVCpu, paParam[i]);
5282
5283 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285 /* Call the switcher. */
5286 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5287 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5288
5289 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5290 /* Make sure the VMX instructions don't cause #UD faults. */
5291 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5292
5293 /* Re-enter VMX Root Mode */
5294 int rc2 = VMXEnable(HCPhysCpuPage);
5295 if (RT_FAILURE(rc2))
5296 {
5297 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5298 ASMSetFlags(fOldEFlags);
5299 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5300 return rc2;
5301 }
5302
5303 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5304 AssertRC(rc2);
5305 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5306 Assert(!(ASMGetFlags() & X86_EFL_IF));
5307 ASMSetFlags(fOldEFlags);
5308 return rc;
5309}
5310
5311
5312/**
5313 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5314 * supporting 64-bit guests.
5315 *
5316 * @returns VBox status code.
5317 * @param fResume Whether to VMLAUNCH or VMRESUME.
5318 * @param pCtx Pointer to the guest-CPU context.
5319 * @param pCache Pointer to the VMCS cache.
5320 * @param pVM The cross context VM structure.
5321 * @param pVCpu The cross context virtual CPU structure.
5322 */
5323DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5324{
5325 NOREF(fResume);
5326
5327 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5328 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5329
5330#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5331 pCache->uPos = 1;
5332 pCache->interPD = PGMGetInterPaeCR3(pVM);
5333 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5334#endif
5335
5336#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5337 pCache->TestIn.HCPhysCpuPage = 0;
5338 pCache->TestIn.HCPhysVmcs = 0;
5339 pCache->TestIn.pCache = 0;
5340 pCache->TestOut.HCPhysVmcs = 0;
5341 pCache->TestOut.pCache = 0;
5342 pCache->TestOut.pCtx = 0;
5343 pCache->TestOut.eflags = 0;
5344#else
5345 NOREF(pCache);
5346#endif
5347
5348 uint32_t aParam[10];
5349 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5350 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5351 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5352 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5353 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5354 aParam[5] = 0;
5355 aParam[6] = VM_RC_ADDR(pVM, pVM);
5356 aParam[7] = 0;
5357 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5358 aParam[9] = 0;
5359
5360#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5361 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5362 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5363#endif
5364 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5365
5366#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5367 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5368 Assert(pCtx->dr[4] == 10);
5369 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5370#endif
5371
5372#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5373 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5374 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5375 pVCpu->hm.s.vmx.HCPhysVmcs));
5376 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5377 pCache->TestOut.HCPhysVmcs));
5378 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5379 pCache->TestOut.pCache));
5380 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5381 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5382 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5383 pCache->TestOut.pCtx));
5384 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5385#endif
5386 NOREF(pCtx);
5387 return rc;
5388}
5389
5390
5391/**
5392 * Initialize the VMCS-Read cache.
5393 *
5394 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5395 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5396 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5397 * (those that have a 32-bit FULL & HIGH part).
5398 *
5399 * @returns VBox status code.
5400 * @param pVCpu The cross context virtual CPU structure.
5401 */
5402static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5403{
5404#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5405 do { \
5406 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5407 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5408 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5409 ++cReadFields; \
5410 } while (0)
5411
5412 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5413 uint32_t cReadFields = 0;
5414
5415 /*
5416 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5417 * and serve to indicate exceptions to the rules.
5418 */
5419
5420 /* Guest-natural selector base fields. */
5421#if 0
5422 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5425#endif
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5438#if 0
5439 /* Unused natural width guest-state fields. */
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5442#endif
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5445
5446 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5447 these 64-bit fields (using "FULL" and "HIGH" fields). */
5448#if 0
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5458#endif
5459
5460 /* Natural width guest-state fields. */
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5462#if 0
5463 /* Currently unused field. */
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5465#endif
5466
5467 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5468 {
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5470 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5471 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5472 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5473 }
5474 else
5475 {
5476 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5477 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5478 }
5479
5480#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5481 return VINF_SUCCESS;
5482}
5483
5484
5485/**
5486 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5487 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5488 * darwin, running 64-bit guests).
5489 *
5490 * @returns VBox status code.
5491 * @param pVCpu The cross context virtual CPU structure.
5492 * @param idxField The VMCS field encoding.
5493 * @param u64Val 16, 32 or 64-bit value.
5494 */
5495VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5496{
5497 int rc;
5498 switch (idxField)
5499 {
5500 /*
5501 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5502 */
5503 /* 64-bit Control fields. */
5504 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5505 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5506 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5507 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5508 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5509 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5510 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5511 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5512 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5513 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5514 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5515 case VMX_VMCS64_CTRL_EPTP_FULL:
5516 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5517 /* 64-bit Guest-state fields. */
5518 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5519 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5520 case VMX_VMCS64_GUEST_PAT_FULL:
5521 case VMX_VMCS64_GUEST_EFER_FULL:
5522 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5523 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5524 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5525 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5526 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5527 /* 64-bit Host-state fields. */
5528 case VMX_VMCS64_HOST_PAT_FULL:
5529 case VMX_VMCS64_HOST_EFER_FULL:
5530 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5531 {
5532 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5533 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5534 break;
5535 }
5536
5537 /*
5538 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5539 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5540 */
5541 /* Natural-width Guest-state fields. */
5542 case VMX_VMCS_GUEST_CR3:
5543 case VMX_VMCS_GUEST_ES_BASE:
5544 case VMX_VMCS_GUEST_CS_BASE:
5545 case VMX_VMCS_GUEST_SS_BASE:
5546 case VMX_VMCS_GUEST_DS_BASE:
5547 case VMX_VMCS_GUEST_FS_BASE:
5548 case VMX_VMCS_GUEST_GS_BASE:
5549 case VMX_VMCS_GUEST_LDTR_BASE:
5550 case VMX_VMCS_GUEST_TR_BASE:
5551 case VMX_VMCS_GUEST_GDTR_BASE:
5552 case VMX_VMCS_GUEST_IDTR_BASE:
5553 case VMX_VMCS_GUEST_RSP:
5554 case VMX_VMCS_GUEST_RIP:
5555 case VMX_VMCS_GUEST_SYSENTER_ESP:
5556 case VMX_VMCS_GUEST_SYSENTER_EIP:
5557 {
5558 if (!(RT_HI_U32(u64Val)))
5559 {
5560 /* If this field is 64-bit, VT-x will zero out the top bits. */
5561 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5562 }
5563 else
5564 {
5565 /* Assert that only the 32->64 switcher case should ever come here. */
5566 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5567 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5568 }
5569 break;
5570 }
5571
5572 default:
5573 {
5574 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5575 rc = VERR_INVALID_PARAMETER;
5576 break;
5577 }
5578 }
5579 AssertRCReturn(rc, rc);
5580 return rc;
5581}
5582
5583
5584/**
5585 * Queue up a VMWRITE by using the VMCS write cache.
5586 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5587 *
5588 * @param pVCpu The cross context virtual CPU structure.
5589 * @param idxField The VMCS field encoding.
5590 * @param u64Val 16, 32 or 64-bit value.
5591 */
5592VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5593{
5594 AssertPtr(pVCpu);
5595 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5596
5597 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5598 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5599
5600 /* Make sure there are no duplicates. */
5601 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5602 {
5603 if (pCache->Write.aField[i] == idxField)
5604 {
5605 pCache->Write.aFieldVal[i] = u64Val;
5606 return VINF_SUCCESS;
5607 }
5608 }
5609
5610 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5611 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5612 pCache->Write.cValidEntries++;
5613 return VINF_SUCCESS;
5614}
5615#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5616
5617
5618/**
5619 * Sets up the usage of TSC-offsetting and updates the VMCS.
5620 *
5621 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5622 * VMX preemption timer.
5623 *
5624 * @returns VBox status code.
5625 * @param pVCpu The cross context virtual CPU structure.
5626 *
5627 * @remarks No-long-jump zone!!!
5628 */
5629static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5630{
5631 bool fOffsettedTsc;
5632 bool fParavirtTsc;
5633 PVM pVM = pVCpu->CTX_SUFF(pVM);
5634 uint64_t uTscOffset;
5635 if (pVM->hm.s.vmx.fUsePreemptTimer)
5636 {
5637 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5638
5639 /* Make sure the returned values have sane upper and lower boundaries. */
5640 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5641 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5642 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5643 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5644
5645 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5646 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5647 AssertRC(rc);
5648 }
5649 else
5650 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5651
5652 /** @todo later optimize this to be done elsewhere and not before every
5653 * VM-entry. */
5654 if (fParavirtTsc)
5655 {
5656 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5657 information before every VM-entry, hence disable it for performance sake. */
5658#if 0
5659 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5660 AssertRC(rc);
5661#endif
5662 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5663 }
5664
5665 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5666 if ( fOffsettedTsc
5667 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5668 {
5669 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5670 {
5671 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5672 AssertRC(rc);
5673 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5674 }
5675
5676 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5677 {
5678 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5679 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5680 AssertRC(rc);
5681 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5682 }
5683 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5684 }
5685 else
5686 {
5687 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5688 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5689 {
5690 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5691 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5692 AssertRC(rc);
5693 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5694 }
5695 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5696 }
5697}
5698
5699
5700/**
5701 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5702 * VM-exit interruption info type.
5703 *
5704 * @returns The IEM exception flags.
5705 * @param uVector The event vector.
5706 * @param uVmxVectorType The VMX event type.
5707 *
5708 * @remarks This function currently only constructs flags required for
5709 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5710 * and CR2 aspects of an exception are not included).
5711 */
5712static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5713{
5714 uint32_t fIemXcptFlags;
5715 switch (uVmxVectorType)
5716 {
5717 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5718 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5719 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5720 break;
5721
5722 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5724 break;
5725
5726 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5727 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5728 break;
5729
5730 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5731 {
5732 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5733 if (uVector == X86_XCPT_BP)
5734 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5735 else if (uVector == X86_XCPT_OF)
5736 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5737 else
5738 {
5739 fIemXcptFlags = 0;
5740 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5741 }
5742 break;
5743 }
5744
5745 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5746 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5747 break;
5748
5749 default:
5750 fIemXcptFlags = 0;
5751 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5752 break;
5753 }
5754 return fIemXcptFlags;
5755}
5756
5757
5758/**
5759 * Sets an event as a pending event to be injected into the guest.
5760 *
5761 * @param pVCpu The cross context virtual CPU structure.
5762 * @param u32IntInfo The VM-entry interruption-information field.
5763 * @param cbInstr The VM-entry instruction length in bytes (for software
5764 * interrupts, exceptions and privileged software
5765 * exceptions).
5766 * @param u32ErrCode The VM-entry exception error code.
5767 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5768 * page-fault.
5769 *
5770 * @remarks Statistics counter assumes this is a guest event being injected or
5771 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5772 * always incremented.
5773 */
5774DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5775 RTGCUINTPTR GCPtrFaultAddress)
5776{
5777 Assert(!pVCpu->hm.s.Event.fPending);
5778 pVCpu->hm.s.Event.fPending = true;
5779 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5780 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5781 pVCpu->hm.s.Event.cbInstr = cbInstr;
5782 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5783}
5784
5785
5786/**
5787 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5788 *
5789 * @param pVCpu The cross context virtual CPU structure.
5790 */
5791DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5792{
5793 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5795 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5796 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5797}
5798
5799
5800/**
5801 * Handle a condition that occurred while delivering an event through the guest
5802 * IDT.
5803 *
5804 * @returns Strict VBox status code (i.e. informational status codes too).
5805 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5806 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5807 * to continue execution of the guest which will delivery the \#DF.
5808 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5809 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5810 *
5811 * @param pVCpu The cross context virtual CPU structure.
5812 * @param pVmxTransient Pointer to the VMX transient structure.
5813 *
5814 * @remarks No-long-jump zone!!!
5815 */
5816static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5817{
5818 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5819
5820 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5821 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5822 AssertRCReturn(rc2, rc2);
5823
5824 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5825 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5826 {
5827 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5828 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5829
5830 /*
5831 * If the event was a software interrupt (generated with INT n) or a software exception
5832 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5833 * can handle the VM-exit and continue guest execution which will re-execute the
5834 * instruction rather than re-injecting the exception, as that can cause premature
5835 * trips to ring-3 before injection and involve TRPM which currently has no way of
5836 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5837 * the problem).
5838 */
5839 IEMXCPTRAISE enmRaise;
5840 IEMXCPTRAISEINFO fRaiseInfo;
5841 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5842 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5843 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5844 {
5845 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5846 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5847 }
5848 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5849 {
5850 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5851 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5852 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5853 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5854 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5855 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5856 uExitVectorType), VERR_VMX_IPE_5);
5857
5858 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5859
5860 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5861 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5862 {
5863 pVmxTransient->fVectoringPF = true;
5864 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5865 }
5866 }
5867 else
5868 {
5869 /*
5870 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5871 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5872 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5873 */
5874 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5875 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5876 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5877 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5878 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5879 }
5880
5881 /*
5882 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5883 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5884 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5885 * subsequent VM-entry would fail.
5886 *
5887 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5888 */
5889 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5890 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5891 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5892 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5893 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5894 {
5895 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5896 }
5897
5898 switch (enmRaise)
5899 {
5900 case IEMXCPTRAISE_CURRENT_XCPT:
5901 {
5902 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5903 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5904 Assert(rcStrict == VINF_SUCCESS);
5905 break;
5906 }
5907
5908 case IEMXCPTRAISE_PREV_EVENT:
5909 {
5910 uint32_t u32ErrCode;
5911 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5912 {
5913 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5914 AssertRCReturn(rc2, rc2);
5915 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5916 }
5917 else
5918 u32ErrCode = 0;
5919
5920 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5921 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5922 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5923 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5924
5925 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5926 pVCpu->hm.s.Event.u32ErrCode));
5927 Assert(rcStrict == VINF_SUCCESS);
5928 break;
5929 }
5930
5931 case IEMXCPTRAISE_REEXEC_INSTR:
5932 Assert(rcStrict == VINF_SUCCESS);
5933 break;
5934
5935 case IEMXCPTRAISE_DOUBLE_FAULT:
5936 {
5937 /*
5938 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5939 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5940 */
5941 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5942 {
5943 pVmxTransient->fVectoringDoublePF = true;
5944 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5945 pVCpu->cpum.GstCtx.cr2));
5946 rcStrict = VINF_SUCCESS;
5947 }
5948 else
5949 {
5950 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5951 hmR0VmxSetPendingXcptDF(pVCpu);
5952 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5953 uIdtVector, uExitVector));
5954 rcStrict = VINF_HM_DOUBLE_FAULT;
5955 }
5956 break;
5957 }
5958
5959 case IEMXCPTRAISE_TRIPLE_FAULT:
5960 {
5961 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5962 rcStrict = VINF_EM_RESET;
5963 break;
5964 }
5965
5966 case IEMXCPTRAISE_CPU_HANG:
5967 {
5968 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5969 rcStrict = VERR_EM_GUEST_CPU_HANG;
5970 break;
5971 }
5972
5973 default:
5974 {
5975 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5976 rcStrict = VERR_VMX_IPE_2;
5977 break;
5978 }
5979 }
5980 }
5981 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5982 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5983 && uExitVector != X86_XCPT_DF
5984 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5985 {
5986 /*
5987 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5988 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5989 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5990 */
5991 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5992 {
5993 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5994 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5995 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5996 }
5997 }
5998
5999 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6000 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6001 return rcStrict;
6002}
6003
6004
6005/**
6006 * Imports a guest segment register from the current VMCS into
6007 * the guest-CPU context.
6008 *
6009 * @returns VBox status code.
6010 * @param pVCpu The cross context virtual CPU structure.
6011 * @param idxSel Index of the selector in the VMCS.
6012 * @param idxLimit Index of the segment limit in the VMCS.
6013 * @param idxBase Index of the segment base in the VMCS.
6014 * @param idxAccess Index of the access rights of the segment in the VMCS.
6015 * @param pSelReg Pointer to the segment selector.
6016 *
6017 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6018 * do not log!
6019 *
6020 * @remarks Never call this function directly!!! Use the
6021 * HMVMX_IMPORT_SREG() macro as that takes care
6022 * of whether to read from the VMCS cache or not.
6023 */
6024static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6025 PCPUMSELREG pSelReg)
6026{
6027 NOREF(pVCpu);
6028
6029 uint32_t u32Sel;
6030 uint32_t u32Limit;
6031 uint32_t u32Attr;
6032 uint64_t u64Base;
6033 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6034 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6035 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6036 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6037 AssertRCReturn(rc, rc);
6038
6039 pSelReg->Sel = (uint16_t)u32Sel;
6040 pSelReg->ValidSel = (uint16_t)u32Sel;
6041 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6042 pSelReg->u32Limit = u32Limit;
6043 pSelReg->u64Base = u64Base;
6044 pSelReg->Attr.u = u32Attr;
6045
6046 /*
6047 * If VT-x marks the segment as unusable, most other bits remain undefined:
6048 * - For CS the L, D and G bits have meaning.
6049 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6050 * - For the remaining data segments no bits are defined.
6051 *
6052 * The present bit and the unusable bit has been observed to be set at the
6053 * same time (the selector was supposed to be invalid as we started executing
6054 * a V8086 interrupt in ring-0).
6055 *
6056 * What should be important for the rest of the VBox code, is that the P bit is
6057 * cleared. Some of the other VBox code recognizes the unusable bit, but
6058 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6059 * safe side here, we'll strip off P and other bits we don't care about. If
6060 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6061 *
6062 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6063 */
6064 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6065 {
6066 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6067
6068 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6069 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6070 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6071#ifdef VBOX_STRICT
6072 VMMRZCallRing3Disable(pVCpu);
6073 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6074# ifdef DEBUG_bird
6075 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6076 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6077 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6078# endif
6079 VMMRZCallRing3Enable(pVCpu);
6080#endif
6081 }
6082 return VINF_SUCCESS;
6083}
6084
6085
6086/**
6087 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6088 *
6089 * @returns VBox status code.
6090 * @param pVCpu The cross context virtual CPU structure.
6091 *
6092 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6093 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6094 * instead!!!
6095 */
6096DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6097{
6098 uint64_t u64Val;
6099 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6100 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6101 {
6102 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6103 if (RT_SUCCESS(rc))
6104 {
6105 pCtx->rip = u64Val;
6106 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6107 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6108 }
6109 return rc;
6110 }
6111 return VINF_SUCCESS;
6112}
6113
6114
6115/**
6116 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6117 *
6118 * @returns VBox status code.
6119 * @param pVCpu The cross context virtual CPU structure.
6120 *
6121 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6122 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6123 * instead!!!
6124 */
6125DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6126{
6127 uint32_t u32Val;
6128 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6129 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6130 {
6131 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6132 if (RT_SUCCESS(rc))
6133 {
6134 pCtx->eflags.u32 = u32Val;
6135
6136 /* Restore eflags for real-on-v86-mode hack. */
6137 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6138 {
6139 pCtx->eflags.Bits.u1VM = 0;
6140 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6141 }
6142 }
6143 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6144 return rc;
6145 }
6146 return VINF_SUCCESS;
6147}
6148
6149
6150/**
6151 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6152 * context.
6153 *
6154 * @returns VBox status code.
6155 * @param pVCpu The cross context virtual CPU structure.
6156 *
6157 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6158 * do not log!
6159 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6160 * instead!!!
6161 */
6162DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6163{
6164 uint32_t u32Val;
6165 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6166 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6167 if (RT_SUCCESS(rc))
6168 {
6169 /*
6170 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6171 * might need them in hmR0VmxEvaluatePendingEvent().
6172 */
6173 if (!u32Val)
6174 {
6175 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6176 {
6177 rc = hmR0VmxImportGuestRip(pVCpu);
6178 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6179 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6180 }
6181
6182 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6183 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6184 }
6185 else
6186 {
6187 rc = hmR0VmxImportGuestRip(pVCpu);
6188 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6189
6190 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6191 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6192 {
6193 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6194 }
6195 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6196 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6197
6198 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6199 {
6200 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6201 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6202 }
6203 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6204 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6205 }
6206 }
6207 return rc;
6208}
6209
6210
6211/**
6212 * Worker for VMXR0ImportStateOnDemand.
6213 *
6214 * @returns VBox status code.
6215 * @param pVCpu The cross context virtual CPU structure.
6216 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6217 */
6218static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6219{
6220#define VMXLOCAL_BREAK_RC(a_rc) \
6221 if (RT_FAILURE(a_rc)) \
6222 break
6223
6224 int rc = VINF_SUCCESS;
6225 PVM pVM = pVCpu->CTX_SUFF(pVM);
6226 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6227 uint64_t u64Val;
6228 uint32_t u32Val;
6229
6230 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6231 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6232
6233 /*
6234 * We disable interrupts to make the updating of the state and in particular
6235 * the fExtrn modification atomic wrt to preemption hooks.
6236 */
6237 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6238
6239 fWhat &= pCtx->fExtrn;
6240 if (fWhat)
6241 {
6242 do
6243 {
6244 if (fWhat & CPUMCTX_EXTRN_RIP)
6245 {
6246 rc = hmR0VmxImportGuestRip(pVCpu);
6247 VMXLOCAL_BREAK_RC(rc);
6248 }
6249
6250 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6251 {
6252 rc = hmR0VmxImportGuestRFlags(pVCpu);
6253 VMXLOCAL_BREAK_RC(rc);
6254 }
6255
6256 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6257 {
6258 rc = hmR0VmxImportGuestIntrState(pVCpu);
6259 VMXLOCAL_BREAK_RC(rc);
6260 }
6261
6262 if (fWhat & CPUMCTX_EXTRN_RSP)
6263 {
6264 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6265 VMXLOCAL_BREAK_RC(rc);
6266 pCtx->rsp = u64Val;
6267 }
6268
6269 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6270 {
6271 if (fWhat & CPUMCTX_EXTRN_CS)
6272 {
6273 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6274 rc |= hmR0VmxImportGuestRip(pVCpu);
6275 VMXLOCAL_BREAK_RC(rc);
6276 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6277 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6278 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6279 }
6280 if (fWhat & CPUMCTX_EXTRN_SS)
6281 {
6282 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6283 VMXLOCAL_BREAK_RC(rc);
6284 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6285 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6286 }
6287 if (fWhat & CPUMCTX_EXTRN_DS)
6288 {
6289 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6290 VMXLOCAL_BREAK_RC(rc);
6291 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6292 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6293 }
6294 if (fWhat & CPUMCTX_EXTRN_ES)
6295 {
6296 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6297 VMXLOCAL_BREAK_RC(rc);
6298 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6299 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6300 }
6301 if (fWhat & CPUMCTX_EXTRN_FS)
6302 {
6303 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6304 VMXLOCAL_BREAK_RC(rc);
6305 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6306 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6307 }
6308 if (fWhat & CPUMCTX_EXTRN_GS)
6309 {
6310 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6311 VMXLOCAL_BREAK_RC(rc);
6312 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6313 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6314 }
6315 }
6316
6317 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6318 {
6319 if (fWhat & CPUMCTX_EXTRN_LDTR)
6320 {
6321 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6322 VMXLOCAL_BREAK_RC(rc);
6323 }
6324
6325 if (fWhat & CPUMCTX_EXTRN_GDTR)
6326 {
6327 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6328 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6329 VMXLOCAL_BREAK_RC(rc);
6330 pCtx->gdtr.pGdt = u64Val;
6331 pCtx->gdtr.cbGdt = u32Val;
6332 }
6333
6334 /* Guest IDTR. */
6335 if (fWhat & CPUMCTX_EXTRN_IDTR)
6336 {
6337 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6338 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6339 VMXLOCAL_BREAK_RC(rc);
6340 pCtx->idtr.pIdt = u64Val;
6341 pCtx->idtr.cbIdt = u32Val;
6342 }
6343
6344 /* Guest TR. */
6345 if (fWhat & CPUMCTX_EXTRN_TR)
6346 {
6347 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6348 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6349 {
6350 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6351 VMXLOCAL_BREAK_RC(rc);
6352 }
6353 }
6354 }
6355
6356 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6357 {
6358 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6359 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6360 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6361 pCtx->SysEnter.cs = u32Val;
6362 VMXLOCAL_BREAK_RC(rc);
6363 }
6364
6365#if HC_ARCH_BITS == 64
6366 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6367 {
6368 if ( pVM->hm.s.fAllow64BitGuests
6369 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6370 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6371 }
6372
6373 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6374 {
6375 if ( pVM->hm.s.fAllow64BitGuests
6376 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6377 {
6378 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6379 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6380 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6381 }
6382 }
6383#endif
6384
6385 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6386#if HC_ARCH_BITS == 32
6387 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6388#endif
6389 )
6390 {
6391 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6392 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6393 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6394 {
6395 switch (pMsr->u32Msr)
6396 {
6397#if HC_ARCH_BITS == 32
6398 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6399 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6400 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6401 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6402#endif
6403 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6404 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6405 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6406 default:
6407 {
6408 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6409 ASMSetFlags(fEFlags);
6410 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6411 cMsrs));
6412 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6413 }
6414 }
6415 }
6416 }
6417
6418 if (fWhat & CPUMCTX_EXTRN_DR7)
6419 {
6420 if (!pVCpu->hm.s.fUsingHyperDR7)
6421 {
6422 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6423 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6424 VMXLOCAL_BREAK_RC(rc);
6425 pCtx->dr[7] = u32Val;
6426 }
6427 }
6428
6429 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6430 {
6431 uint32_t u32Shadow;
6432 if (fWhat & CPUMCTX_EXTRN_CR0)
6433 {
6434 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6435 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6436 VMXLOCAL_BREAK_RC(rc);
6437 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6438 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6439 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6440 CPUMSetGuestCR0(pVCpu, u32Val);
6441 VMMRZCallRing3Enable(pVCpu);
6442 }
6443
6444 if (fWhat & CPUMCTX_EXTRN_CR4)
6445 {
6446 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6447 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6448 VMXLOCAL_BREAK_RC(rc);
6449 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6450 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6451 CPUMSetGuestCR4(pVCpu, u32Val);
6452 }
6453
6454 if (fWhat & CPUMCTX_EXTRN_CR3)
6455 {
6456 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6457 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6458 || ( pVM->hm.s.fNestedPaging
6459 && CPUMIsGuestPagingEnabledEx(pCtx)))
6460 {
6461 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6462 if (pCtx->cr3 != u64Val)
6463 {
6464 CPUMSetGuestCR3(pVCpu, u64Val);
6465 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6466 }
6467
6468 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6469 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6470 if (CPUMIsGuestInPAEModeEx(pCtx))
6471 {
6472 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6473 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6474 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6475 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6476 VMXLOCAL_BREAK_RC(rc);
6477 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6478 }
6479 }
6480 }
6481 }
6482 } while (0);
6483
6484 if (RT_SUCCESS(rc))
6485 {
6486 /* Update fExtrn. */
6487 pCtx->fExtrn &= ~fWhat;
6488
6489 /* If everything has been imported, clear the HM keeper bit. */
6490 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6491 {
6492 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6493 Assert(!pCtx->fExtrn);
6494 }
6495 }
6496 }
6497 else
6498 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6499
6500 ASMSetFlags(fEFlags);
6501
6502 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6503
6504 /*
6505 * Honor any pending CR3 updates.
6506 *
6507 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6508 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6509 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6510 *
6511 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6512 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6513 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6514 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6515 *
6516 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6517 */
6518 if (VMMRZCallRing3IsEnabled(pVCpu))
6519 {
6520 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6521 {
6522 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6523 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6524 }
6525
6526 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6527 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6528
6529 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6530 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6531 }
6532
6533 return VINF_SUCCESS;
6534#undef VMXLOCAL_BREAK_RC
6535}
6536
6537
6538/**
6539 * Saves the guest state from the VMCS into the guest-CPU context.
6540 *
6541 * @returns VBox status code.
6542 * @param pVCpu The cross context virtual CPU structure.
6543 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6544 */
6545VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6546{
6547 return hmR0VmxImportGuestState(pVCpu, fWhat);
6548}
6549
6550
6551/**
6552 * Check per-VM and per-VCPU force flag actions that require us to go back to
6553 * ring-3 for one reason or another.
6554 *
6555 * @returns Strict VBox status code (i.e. informational status codes too)
6556 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6557 * ring-3.
6558 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6559 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6560 * interrupts)
6561 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6562 * all EMTs to be in ring-3.
6563 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6564 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6565 * to the EM loop.
6566 *
6567 * @param pVCpu The cross context virtual CPU structure.
6568 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6569 */
6570static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6571{
6572 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6573
6574 /*
6575 * Anything pending? Should be more likely than not if we're doing a good job.
6576 */
6577 PVM pVM = pVCpu->CTX_SUFF(pVM);
6578 if ( !fStepping
6579 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6580 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6581 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6582 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6583 return VINF_SUCCESS;
6584
6585 /* Pending PGM C3 sync. */
6586 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6587 {
6588 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6589 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6590 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6591 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6592 if (rcStrict2 != VINF_SUCCESS)
6593 {
6594 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6595 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6596 return rcStrict2;
6597 }
6598 }
6599
6600 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6601 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6602 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6603 {
6604 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6605 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6606 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6607 return rc2;
6608 }
6609
6610 /* Pending VM request packets, such as hardware interrupts. */
6611 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6612 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6613 {
6614 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6615 return VINF_EM_PENDING_REQUEST;
6616 }
6617
6618 /* Pending PGM pool flushes. */
6619 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6620 {
6621 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6622 return VINF_PGM_POOL_FLUSH_PENDING;
6623 }
6624
6625 /* Pending DMA requests. */
6626 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6627 {
6628 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6629 return VINF_EM_RAW_TO_R3;
6630 }
6631
6632 return VINF_SUCCESS;
6633}
6634
6635
6636/**
6637 * Converts any TRPM trap into a pending HM event. This is typically used when
6638 * entering from ring-3 (not longjmp returns).
6639 *
6640 * @param pVCpu The cross context virtual CPU structure.
6641 */
6642static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6643{
6644 Assert(TRPMHasTrap(pVCpu));
6645 Assert(!pVCpu->hm.s.Event.fPending);
6646
6647 uint8_t uVector;
6648 TRPMEVENT enmTrpmEvent;
6649 RTGCUINT uErrCode;
6650 RTGCUINTPTR GCPtrFaultAddress;
6651 uint8_t cbInstr;
6652
6653 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6654 AssertRC(rc);
6655
6656 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6657 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6658 if (enmTrpmEvent == TRPM_TRAP)
6659 {
6660 switch (uVector)
6661 {
6662 case X86_XCPT_NMI:
6663 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6664 break;
6665
6666 case X86_XCPT_BP:
6667 case X86_XCPT_OF:
6668 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6669 break;
6670
6671 case X86_XCPT_PF:
6672 case X86_XCPT_DF:
6673 case X86_XCPT_TS:
6674 case X86_XCPT_NP:
6675 case X86_XCPT_SS:
6676 case X86_XCPT_GP:
6677 case X86_XCPT_AC:
6678 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6679 RT_FALL_THRU();
6680 default:
6681 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6682 break;
6683 }
6684 }
6685 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6686 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6687 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6688 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6689 else
6690 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6691
6692 rc = TRPMResetTrap(pVCpu);
6693 AssertRC(rc);
6694 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6695 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6696
6697 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6698}
6699
6700
6701/**
6702 * Converts the pending HM event into a TRPM trap.
6703 *
6704 * @param pVCpu The cross context virtual CPU structure.
6705 */
6706static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6707{
6708 Assert(pVCpu->hm.s.Event.fPending);
6709
6710 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6711 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6712 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6713 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6714
6715 /* If a trap was already pending, we did something wrong! */
6716 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6717
6718 TRPMEVENT enmTrapType;
6719 switch (uVectorType)
6720 {
6721 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6722 enmTrapType = TRPM_HARDWARE_INT;
6723 break;
6724
6725 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6726 enmTrapType = TRPM_SOFTWARE_INT;
6727 break;
6728
6729 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6730 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6731 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6732 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6733 enmTrapType = TRPM_TRAP;
6734 break;
6735
6736 default:
6737 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6738 enmTrapType = TRPM_32BIT_HACK;
6739 break;
6740 }
6741
6742 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6743
6744 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6745 AssertRC(rc);
6746
6747 if (fErrorCodeValid)
6748 TRPMSetErrorCode(pVCpu, uErrorCode);
6749
6750 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6751 && uVector == X86_XCPT_PF)
6752 {
6753 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6754 }
6755 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6756 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6757 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6758 {
6759 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6760 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6761 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6762 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6763 }
6764
6765 /* Clear any pending events from the VMCS. */
6766 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6767 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6768
6769 /* We're now done converting the pending event. */
6770 pVCpu->hm.s.Event.fPending = false;
6771}
6772
6773
6774/**
6775 * Does the necessary state syncing before returning to ring-3 for any reason
6776 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6777 *
6778 * @returns VBox status code.
6779 * @param pVCpu The cross context virtual CPU structure.
6780 * @param fImportState Whether to import the guest state from the VMCS back
6781 * to the guest-CPU context.
6782 *
6783 * @remarks No-long-jmp zone!!!
6784 */
6785static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6786{
6787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6788 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6789
6790 RTCPUID idCpu = RTMpCpuId();
6791 Log4Func(("HostCpuId=%u\n", idCpu));
6792
6793 /*
6794 * !!! IMPORTANT !!!
6795 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6796 */
6797
6798 /* Save the guest state if necessary. */
6799 if (fImportState)
6800 {
6801 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6802 AssertRCReturn(rc, rc);
6803 }
6804
6805 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6806 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6807 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6808
6809 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6810#ifdef VBOX_STRICT
6811 if (CPUMIsHyperDebugStateActive(pVCpu))
6812 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6813#endif
6814 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6815 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6816 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6817
6818#if HC_ARCH_BITS == 64
6819 /* Restore host-state bits that VT-x only restores partially. */
6820 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6821 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6822 {
6823 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6824 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6825 }
6826 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6827#endif
6828
6829 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6830 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6831 {
6832 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6833 if (!fImportState)
6834 {
6835 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6836 AssertRCReturn(rc, rc);
6837 }
6838 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6839 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6840 }
6841 else
6842 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6843
6844 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6845 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6846
6847 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6848 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6849 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6850 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6851 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6852 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6853 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6854 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6855 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6856
6857 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6858
6859 /** @todo This partially defeats the purpose of having preemption hooks.
6860 * The problem is, deregistering the hooks should be moved to a place that
6861 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6862 * context.
6863 */
6864 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6865 {
6866 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6867 AssertRCReturn(rc, rc);
6868
6869 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6870 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6871 }
6872 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6873 NOREF(idCpu);
6874
6875 return VINF_SUCCESS;
6876}
6877
6878
6879/**
6880 * Leaves the VT-x session.
6881 *
6882 * @returns VBox status code.
6883 * @param pVCpu The cross context virtual CPU structure.
6884 *
6885 * @remarks No-long-jmp zone!!!
6886 */
6887static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6888{
6889 HM_DISABLE_PREEMPT(pVCpu);
6890 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6891 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6892 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6893
6894 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6895 and done this from the VMXR0ThreadCtxCallback(). */
6896 if (!pVCpu->hm.s.fLeaveDone)
6897 {
6898 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6899 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6900 pVCpu->hm.s.fLeaveDone = true;
6901 }
6902 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6903
6904 /*
6905 * !!! IMPORTANT !!!
6906 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6907 */
6908
6909 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6910 /** @todo Deregistering here means we need to VMCLEAR always
6911 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6912 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6913 VMMR0ThreadCtxHookDisable(pVCpu);
6914
6915 /* Leave HM context. This takes care of local init (term). */
6916 int rc = HMR0LeaveCpu(pVCpu);
6917
6918 HM_RESTORE_PREEMPT();
6919 return rc;
6920}
6921
6922
6923/**
6924 * Does the necessary state syncing before doing a longjmp to ring-3.
6925 *
6926 * @returns VBox status code.
6927 * @param pVCpu The cross context virtual CPU structure.
6928 *
6929 * @remarks No-long-jmp zone!!!
6930 */
6931DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6932{
6933 return hmR0VmxLeaveSession(pVCpu);
6934}
6935
6936
6937/**
6938 * Take necessary actions before going back to ring-3.
6939 *
6940 * An action requires us to go back to ring-3. This function does the necessary
6941 * steps before we can safely return to ring-3. This is not the same as longjmps
6942 * to ring-3, this is voluntary and prepares the guest so it may continue
6943 * executing outside HM (recompiler/IEM).
6944 *
6945 * @returns VBox status code.
6946 * @param pVCpu The cross context virtual CPU structure.
6947 * @param rcExit The reason for exiting to ring-3. Can be
6948 * VINF_VMM_UNKNOWN_RING3_CALL.
6949 */
6950static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6951{
6952 Assert(pVCpu);
6953 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6954
6955 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6956 {
6957 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6958 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6959 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6960 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6961 }
6962
6963 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6964 VMMRZCallRing3Disable(pVCpu);
6965 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6966
6967 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6968 if (pVCpu->hm.s.Event.fPending)
6969 {
6970 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6971 Assert(!pVCpu->hm.s.Event.fPending);
6972 }
6973
6974 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6975 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6976
6977 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6978 and if we're injecting an event we should have a TRPM trap pending. */
6979 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6980#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6981 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6982#endif
6983
6984 /* Save guest state and restore host state bits. */
6985 int rc = hmR0VmxLeaveSession(pVCpu);
6986 AssertRCReturn(rc, rc);
6987 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6988 /* Thread-context hooks are unregistered at this point!!! */
6989
6990 /* Sync recompiler state. */
6991 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6992 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6993 | CPUM_CHANGED_LDTR
6994 | CPUM_CHANGED_GDTR
6995 | CPUM_CHANGED_IDTR
6996 | CPUM_CHANGED_TR
6997 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6998 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6999 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7000 {
7001 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7002 }
7003
7004 Assert(!pVCpu->hm.s.fClearTrapFlag);
7005
7006 /* Update the exit-to-ring 3 reason. */
7007 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7008
7009 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7010 if (rcExit != VINF_EM_RAW_INTERRUPT)
7011 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7012
7013 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7014
7015 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7016 VMMRZCallRing3RemoveNotification(pVCpu);
7017 VMMRZCallRing3Enable(pVCpu);
7018
7019 return rc;
7020}
7021
7022
7023/**
7024 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7025 * longjump to ring-3 and possibly get preempted.
7026 *
7027 * @returns VBox status code.
7028 * @param pVCpu The cross context virtual CPU structure.
7029 * @param enmOperation The operation causing the ring-3 longjump.
7030 * @param pvUser User argument, currently unused, NULL.
7031 */
7032static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7033{
7034 RT_NOREF(pvUser);
7035 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7036 {
7037 /*
7038 * !!! IMPORTANT !!!
7039 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7040 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7041 */
7042 VMMRZCallRing3RemoveNotification(pVCpu);
7043 VMMRZCallRing3Disable(pVCpu);
7044 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7045 RTThreadPreemptDisable(&PreemptState);
7046
7047 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7048 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7049 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7050
7051#if HC_ARCH_BITS == 64
7052 /* Restore host-state bits that VT-x only restores partially. */
7053 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7054 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7055 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7056 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7057#endif
7058
7059 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7060 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7061 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7062
7063 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7064 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7065 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7066 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7067 {
7068 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7069 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7070 }
7071
7072 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7073 VMMR0ThreadCtxHookDisable(pVCpu);
7074 HMR0LeaveCpu(pVCpu);
7075 RTThreadPreemptRestore(&PreemptState);
7076 return VINF_SUCCESS;
7077 }
7078
7079 Assert(pVCpu);
7080 Assert(pvUser);
7081 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7082 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7083
7084 VMMRZCallRing3Disable(pVCpu);
7085 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7086
7087 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7088
7089 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7090 AssertRCReturn(rc, rc);
7091
7092 VMMRZCallRing3Enable(pVCpu);
7093 return VINF_SUCCESS;
7094}
7095
7096
7097/**
7098 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7099 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7100 *
7101 * @param pVCpu The cross context virtual CPU structure.
7102 */
7103DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7104{
7105 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7106 {
7107 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7108 {
7109 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7110 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7111 AssertRC(rc);
7112 Log4Func(("Setup interrupt-window exiting\n"));
7113 }
7114 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7115}
7116
7117
7118/**
7119 * Clears the interrupt-window exiting control in the VMCS.
7120 *
7121 * @param pVCpu The cross context virtual CPU structure.
7122 */
7123DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7124{
7125 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7126 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7127 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7128 AssertRC(rc);
7129 Log4Func(("Cleared interrupt-window exiting\n"));
7130}
7131
7132
7133/**
7134 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7135 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7136 *
7137 * @param pVCpu The cross context virtual CPU structure.
7138 */
7139DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7140{
7141 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7142 {
7143 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7144 {
7145 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7146 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7147 AssertRC(rc);
7148 Log4Func(("Setup NMI-window exiting\n"));
7149 }
7150 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7151}
7152
7153
7154/**
7155 * Clears the NMI-window exiting control in the VMCS.
7156 *
7157 * @param pVCpu The cross context virtual CPU structure.
7158 */
7159DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7160{
7161 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7162 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7163 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7164 AssertRC(rc);
7165 Log4Func(("Cleared NMI-window exiting\n"));
7166}
7167
7168
7169/**
7170 * Evaluates the event to be delivered to the guest and sets it as the pending
7171 * event.
7172 *
7173 * @returns The VT-x guest-interruptibility state.
7174 * @param pVCpu The cross context virtual CPU structure.
7175 */
7176static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7177{
7178 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7179 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7180 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7181 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7182 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7183 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7184
7185 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7186 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7187 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7188 Assert(!TRPMHasTrap(pVCpu));
7189
7190 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7191 APICUpdatePendingInterrupts(pVCpu);
7192
7193 /*
7194 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7195 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7196 */
7197 /** @todo SMI. SMIs take priority over NMIs. */
7198 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7199 {
7200 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7201 if ( !pVCpu->hm.s.Event.fPending
7202 && !fBlockNmi
7203 && !fBlockSti
7204 && !fBlockMovSS)
7205 {
7206 Log4Func(("Pending NMI\n"));
7207 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7208 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7209
7210 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7211 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7212 }
7213 else
7214 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7215 }
7216 /*
7217 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7218 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7219 */
7220 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7221 && !pVCpu->hm.s.fSingleInstruction)
7222 {
7223 Assert(!DBGFIsStepping(pVCpu));
7224 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7225 AssertRCReturn(rc, 0);
7226 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7227 if ( !pVCpu->hm.s.Event.fPending
7228 && !fBlockInt
7229 && !fBlockSti
7230 && !fBlockMovSS)
7231 {
7232 uint8_t u8Interrupt;
7233 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7234 if (RT_SUCCESS(rc))
7235 {
7236 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7237 uint32_t u32IntInfo = u8Interrupt
7238 | VMX_EXIT_INTERRUPTION_INFO_VALID
7239 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7240
7241 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7242 }
7243 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7244 {
7245 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7246 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7247 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7248
7249 /*
7250 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7251 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7252 * need to re-set this force-flag here.
7253 */
7254 }
7255 else
7256 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7257 }
7258 else
7259 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7260 }
7261
7262 return fIntrState;
7263}
7264
7265
7266/**
7267 * Sets a pending-debug exception to be delivered to the guest if the guest is
7268 * single-stepping in the VMCS.
7269 *
7270 * @param pVCpu The cross context virtual CPU structure.
7271 */
7272DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7273{
7274 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7275 RT_NOREF(pVCpu);
7276 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7277}
7278
7279
7280/**
7281 * Injects any pending events into the guest if the guest is in a state to
7282 * receive them.
7283 *
7284 * @returns Strict VBox status code (i.e. informational status codes too).
7285 * @param pVCpu The cross context virtual CPU structure.
7286 * @param fIntrState The VT-x guest-interruptibility state.
7287 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7288 * return VINF_EM_DBG_STEPPED if the event was
7289 * dispatched directly.
7290 */
7291static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7292{
7293 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7294 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7295
7296 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7297 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7298
7299 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7300 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7301 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7302 Assert(!TRPMHasTrap(pVCpu));
7303
7304 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7305 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7306 if (pVCpu->hm.s.Event.fPending)
7307 {
7308 /*
7309 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7310 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7311 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7312 *
7313 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7314 */
7315 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7316#ifdef VBOX_STRICT
7317 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7318 {
7319 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7320 Assert(!fBlockInt);
7321 Assert(!fBlockSti);
7322 Assert(!fBlockMovSS);
7323 }
7324 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7325 {
7326 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7327 Assert(!fBlockSti);
7328 Assert(!fBlockMovSS);
7329 Assert(!fBlockNmi);
7330 }
7331#endif
7332 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7333 uIntType));
7334 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7335 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7336 &fIntrState);
7337 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7338
7339 /* Update the interruptibility-state as it could have been changed by
7340 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7341 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7342 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7343
7344 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7345 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7346 else
7347 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7348 }
7349
7350 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7351 if ( fBlockSti
7352 || fBlockMovSS)
7353 {
7354 if (!pVCpu->hm.s.fSingleInstruction)
7355 {
7356 /*
7357 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7358 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7359 * See Intel spec. 27.3.4 "Saving Non-Register State".
7360 */
7361 Assert(!DBGFIsStepping(pVCpu));
7362 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7363 AssertRCReturn(rc, rc);
7364 if (pCtx->eflags.Bits.u1TF)
7365 {
7366 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7367 AssertRCReturn(rc2, rc2);
7368 }
7369 }
7370 else if (pCtx->eflags.Bits.u1TF)
7371 {
7372 /*
7373 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7374 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7375 */
7376 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7377 fIntrState = 0;
7378 }
7379 }
7380
7381 /*
7382 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7383 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7384 */
7385 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7386 AssertRCReturn(rc3, rc3);
7387
7388 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7389 NOREF(fBlockMovSS); NOREF(fBlockSti);
7390 return rcStrict;
7391}
7392
7393
7394/**
7395 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7396 *
7397 * @param pVCpu The cross context virtual CPU structure.
7398 */
7399DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7400{
7401 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7402 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7403}
7404
7405
7406/**
7407 * Injects a double-fault (\#DF) exception into the VM.
7408 *
7409 * @returns Strict VBox status code (i.e. informational status codes too).
7410 * @param pVCpu The cross context virtual CPU structure.
7411 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7412 * and should return VINF_EM_DBG_STEPPED if the event
7413 * is injected directly (register modified by us, not
7414 * by hardware on VM-entry).
7415 * @param pfIntrState Pointer to the current guest interruptibility-state.
7416 * This interruptibility-state will be updated if
7417 * necessary. This cannot not be NULL.
7418 */
7419DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7420{
7421 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7422 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7423 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7424 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7425 pfIntrState);
7426}
7427
7428
7429/**
7430 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7431 *
7432 * @param pVCpu The cross context virtual CPU structure.
7433 */
7434DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7435{
7436 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7437 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7438 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7439}
7440
7441
7442/**
7443 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7444 *
7445 * @param pVCpu The cross context virtual CPU structure.
7446 * @param cbInstr The value of RIP that is to be pushed on the guest
7447 * stack.
7448 */
7449DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7450{
7451 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7452 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7453 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7454}
7455
7456
7457/**
7458 * Injects a general-protection (\#GP) fault into the VM.
7459 *
7460 * @returns Strict VBox status code (i.e. informational status codes too).
7461 * @param pVCpu The cross context virtual CPU structure.
7462 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7463 * mode, i.e. in real-mode it's not valid).
7464 * @param u32ErrorCode The error code associated with the \#GP.
7465 * @param fStepping Whether we're running in
7466 * hmR0VmxRunGuestCodeStep() and should return
7467 * VINF_EM_DBG_STEPPED if the event is injected
7468 * directly (register modified by us, not by
7469 * hardware on VM-entry).
7470 * @param pfIntrState Pointer to the current guest interruptibility-state.
7471 * This interruptibility-state will be updated if
7472 * necessary. This cannot not be NULL.
7473 */
7474DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7475 uint32_t *pfIntrState)
7476{
7477 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7478 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7479 if (fErrorCodeValid)
7480 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7481 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7482 pfIntrState);
7483}
7484
7485
7486/**
7487 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7488 *
7489 * @param pVCpu The cross context virtual CPU structure.
7490 * @param uVector The software interrupt vector number.
7491 * @param cbInstr The value of RIP that is to be pushed on the guest
7492 * stack.
7493 */
7494DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7495{
7496 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7497 if ( uVector == X86_XCPT_BP
7498 || uVector == X86_XCPT_OF)
7499 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7500 else
7501 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7502 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7503}
7504
7505
7506/**
7507 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7508 * stack.
7509 *
7510 * @returns Strict VBox status code (i.e. informational status codes too).
7511 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7512 * @param pVCpu The cross context virtual CPU structure.
7513 * @param uValue The value to push to the guest stack.
7514 */
7515static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7516{
7517 /*
7518 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7519 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7520 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7521 */
7522 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7523 if (pCtx->sp == 1)
7524 return VINF_EM_RESET;
7525 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7526 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7527 AssertRC(rc);
7528 return rc;
7529}
7530
7531
7532/**
7533 * Injects an event into the guest upon VM-entry by updating the relevant fields
7534 * in the VM-entry area in the VMCS.
7535 *
7536 * @returns Strict VBox status code (i.e. informational status codes too).
7537 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7538 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7539 *
7540 * @param pVCpu The cross context virtual CPU structure.
7541 * @param u64IntInfo The VM-entry interruption-information field.
7542 * @param cbInstr The VM-entry instruction length in bytes (for
7543 * software interrupts, exceptions and privileged
7544 * software exceptions).
7545 * @param u32ErrCode The VM-entry exception error code.
7546 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7547 * @param pfIntrState Pointer to the current guest interruptibility-state.
7548 * This interruptibility-state will be updated if
7549 * necessary. This cannot not be NULL.
7550 * @param fStepping Whether we're running in
7551 * hmR0VmxRunGuestCodeStep() and should return
7552 * VINF_EM_DBG_STEPPED if the event is injected
7553 * directly (register modified by us, not by
7554 * hardware on VM-entry).
7555 */
7556static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7557 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7558{
7559 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7560 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7561 Assert(pfIntrState);
7562
7563 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7564 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7565 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7566 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7567
7568#ifdef VBOX_STRICT
7569 /*
7570 * Validate the error-code-valid bit for hardware exceptions.
7571 * No error codes for exceptions in real-mode.
7572 *
7573 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7574 */
7575 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7576 && !CPUMIsGuestInRealModeEx(pCtx))
7577 {
7578 switch (uVector)
7579 {
7580 case X86_XCPT_PF:
7581 case X86_XCPT_DF:
7582 case X86_XCPT_TS:
7583 case X86_XCPT_NP:
7584 case X86_XCPT_SS:
7585 case X86_XCPT_GP:
7586 case X86_XCPT_AC:
7587 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7588 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7589 RT_FALL_THRU();
7590 default:
7591 break;
7592 }
7593 }
7594#endif
7595
7596 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7597 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7598 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7599
7600 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7601
7602 /*
7603 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7604 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7605 * interrupt handler in the (real-mode) guest.
7606 *
7607 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7608 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7609 */
7610 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7611 {
7612 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7613 {
7614 /*
7615 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7616 * set the deliver-error-code bit.
7617 *
7618 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7619 */
7620 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7621 }
7622 else
7623 {
7624 PVM pVM = pVCpu->CTX_SUFF(pVM);
7625 Assert(PDMVmmDevHeapIsEnabled(pVM));
7626 Assert(pVM->hm.s.vmx.pRealModeTSS);
7627
7628 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7629 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7630 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7631 AssertRCReturn(rc2, rc2);
7632
7633 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7634 size_t const cbIdtEntry = sizeof(X86IDTR16);
7635 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7636 {
7637 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7638 if (uVector == X86_XCPT_DF)
7639 return VINF_EM_RESET;
7640
7641 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7642 if (uVector == X86_XCPT_GP)
7643 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7644
7645 /*
7646 * If we're injecting an event with no valid IDT entry, inject a #GP.
7647 * No error codes for exceptions in real-mode.
7648 *
7649 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7650 */
7651 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7652 }
7653
7654 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7655 uint16_t uGuestIp = pCtx->ip;
7656 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7657 {
7658 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7659 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7660 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7661 }
7662 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7663 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7664
7665 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7666 X86IDTR16 IdtEntry;
7667 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7668 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7669 AssertRCReturn(rc2, rc2);
7670
7671 /* Construct the stack frame for the interrupt/exception handler. */
7672 VBOXSTRICTRC rcStrict;
7673 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7674 if (rcStrict == VINF_SUCCESS)
7675 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7676 if (rcStrict == VINF_SUCCESS)
7677 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7678
7679 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7680 if (rcStrict == VINF_SUCCESS)
7681 {
7682 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7683 pCtx->rip = IdtEntry.offSel;
7684 pCtx->cs.Sel = IdtEntry.uSel;
7685 pCtx->cs.ValidSel = IdtEntry.uSel;
7686 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7687 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7688 && uVector == X86_XCPT_PF)
7689 pCtx->cr2 = GCPtrFaultAddress;
7690
7691 /* If any other guest-state bits are changed here, make sure to update
7692 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7693 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7694 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7695 | HM_CHANGED_GUEST_RSP);
7696
7697 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7698 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7699 {
7700 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7701 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7702 Log4Func(("Clearing inhibition due to STI\n"));
7703 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7704 }
7705 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7706 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7707
7708 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7709 it, if we are returning to ring-3 before executing guest code. */
7710 pVCpu->hm.s.Event.fPending = false;
7711
7712 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7713 if (fStepping)
7714 rcStrict = VINF_EM_DBG_STEPPED;
7715 }
7716 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7717 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7718 return rcStrict;
7719 }
7720 }
7721
7722 /* Validate. */
7723 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7724 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7725 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7726
7727 /* Inject. */
7728 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7729 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7730 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7731 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7732 AssertRCReturn(rc, rc);
7733
7734 /* Update CR2. */
7735 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7736 && uVector == X86_XCPT_PF)
7737 pCtx->cr2 = GCPtrFaultAddress;
7738
7739 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7740
7741 return VINF_SUCCESS;
7742}
7743
7744
7745/**
7746 * Clears the interrupt-window exiting control in the VMCS and if necessary
7747 * clears the current event in the VMCS as well.
7748 *
7749 * @returns VBox status code.
7750 * @param pVCpu The cross context virtual CPU structure.
7751 *
7752 * @remarks Use this function only to clear events that have not yet been
7753 * delivered to the guest but are injected in the VMCS!
7754 * @remarks No-long-jump zone!!!
7755 */
7756static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7757{
7758 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7759 {
7760 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7761 Log4Func(("Cleared interrupt widow\n"));
7762 }
7763
7764 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7765 {
7766 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7767 Log4Func(("Cleared interrupt widow\n"));
7768 }
7769}
7770
7771
7772/**
7773 * Enters the VT-x session.
7774 *
7775 * @returns VBox status code.
7776 * @param pVCpu The cross context virtual CPU structure.
7777 * @param pHostCpu Pointer to the global CPU info struct.
7778 */
7779VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7780{
7781 AssertPtr(pVCpu);
7782 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7783 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7784 RT_NOREF(pHostCpu);
7785
7786 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7787 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7788 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7789
7790#ifdef VBOX_STRICT
7791 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7792 RTCCUINTREG uHostCR4 = ASMGetCR4();
7793 if (!(uHostCR4 & X86_CR4_VMXE))
7794 {
7795 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7796 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7797 }
7798#endif
7799
7800 /*
7801 * Load the VCPU's VMCS as the current (and active) one.
7802 */
7803 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7804 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7805 if (RT_FAILURE(rc))
7806 return rc;
7807
7808 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7809 pVCpu->hm.s.fLeaveDone = false;
7810 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7811
7812 return VINF_SUCCESS;
7813}
7814
7815
7816/**
7817 * The thread-context callback (only on platforms which support it).
7818 *
7819 * @param enmEvent The thread-context event.
7820 * @param pVCpu The cross context virtual CPU structure.
7821 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7822 * @thread EMT(pVCpu)
7823 */
7824VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7825{
7826 NOREF(fGlobalInit);
7827
7828 switch (enmEvent)
7829 {
7830 case RTTHREADCTXEVENT_OUT:
7831 {
7832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7833 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7834 VMCPU_ASSERT_EMT(pVCpu);
7835
7836 /* No longjmps (logger flushes, locks) in this fragile context. */
7837 VMMRZCallRing3Disable(pVCpu);
7838 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7839
7840 /*
7841 * Restore host-state (FPU, debug etc.)
7842 */
7843 if (!pVCpu->hm.s.fLeaveDone)
7844 {
7845 /*
7846 * Do -not- import the guest-state here as we might already be in the middle of importing
7847 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7848 */
7849 hmR0VmxLeave(pVCpu, false /* fImportState */);
7850 pVCpu->hm.s.fLeaveDone = true;
7851 }
7852
7853 /* Leave HM context, takes care of local init (term). */
7854 int rc = HMR0LeaveCpu(pVCpu);
7855 AssertRC(rc); NOREF(rc);
7856
7857 /* Restore longjmp state. */
7858 VMMRZCallRing3Enable(pVCpu);
7859 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7860 break;
7861 }
7862
7863 case RTTHREADCTXEVENT_IN:
7864 {
7865 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7866 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7867 VMCPU_ASSERT_EMT(pVCpu);
7868
7869 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7870 VMMRZCallRing3Disable(pVCpu);
7871 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7872
7873 /* Initialize the bare minimum state required for HM. This takes care of
7874 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7875 int rc = hmR0EnterCpu(pVCpu);
7876 AssertRC(rc);
7877 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7878 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7879
7880 /* Load the active VMCS as the current one. */
7881 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7882 {
7883 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7884 AssertRC(rc); NOREF(rc);
7885 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7886 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7887 }
7888 pVCpu->hm.s.fLeaveDone = false;
7889
7890 /* Restore longjmp state. */
7891 VMMRZCallRing3Enable(pVCpu);
7892 break;
7893 }
7894
7895 default:
7896 break;
7897 }
7898}
7899
7900
7901/**
7902 * Exports the host state into the VMCS host-state area.
7903 * Sets up the VM-exit MSR-load area.
7904 *
7905 * The CPU state will be loaded from these fields on every successful VM-exit.
7906 *
7907 * @returns VBox status code.
7908 * @param pVCpu The cross context virtual CPU structure.
7909 *
7910 * @remarks No-long-jump zone!!!
7911 */
7912static int hmR0VmxExportHostState(PVMCPU pVCpu)
7913{
7914 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7915
7916 int rc = VINF_SUCCESS;
7917 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7918 {
7919 rc = hmR0VmxExportHostControlRegs();
7920 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7921
7922 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7923 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7924
7925 rc = hmR0VmxExportHostMsrs(pVCpu);
7926 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7927
7928 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7929 }
7930 return rc;
7931}
7932
7933
7934/**
7935 * Saves the host state in the VMCS host-state.
7936 *
7937 * @returns VBox status code.
7938 * @param pVCpu The cross context virtual CPU structure.
7939 *
7940 * @remarks No-long-jump zone!!!
7941 */
7942VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7943{
7944 AssertPtr(pVCpu);
7945 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7946
7947 /*
7948 * Export the host state here while entering HM context.
7949 * When thread-context hooks are used, we might get preempted and have to re-save the host
7950 * state but most of the time we won't be, so do it here before we disable interrupts.
7951 */
7952 return hmR0VmxExportHostState(pVCpu);
7953}
7954
7955
7956/**
7957 * Exports the guest state into the VMCS guest-state area.
7958 *
7959 * The will typically be done before VM-entry when the guest-CPU state and the
7960 * VMCS state may potentially be out of sync.
7961 *
7962 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7963 * VM-entry controls.
7964 * Sets up the appropriate VMX non-root function to execute guest code based on
7965 * the guest CPU mode.
7966 *
7967 * @returns VBox strict status code.
7968 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7969 * without unrestricted guest access and the VMMDev is not presently
7970 * mapped (e.g. EFI32).
7971 *
7972 * @param pVCpu The cross context virtual CPU structure.
7973 *
7974 * @remarks No-long-jump zone!!!
7975 */
7976static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7977{
7978 AssertPtr(pVCpu);
7979 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7980
7981 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7982
7983 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7984
7985 /* Determine real-on-v86 mode. */
7986 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7987 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7988 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7989 {
7990 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7991 }
7992
7993 /*
7994 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7995 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7996 */
7997 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7998 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7999
8000 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8001 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8002 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8003
8004 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8005 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8006 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8007
8008 rc = hmR0VmxExportGuestCR0(pVCpu);
8009 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8010
8011 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8012 if (rcStrict == VINF_SUCCESS)
8013 { /* likely */ }
8014 else
8015 {
8016 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8017 return rcStrict;
8018 }
8019
8020 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8021 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8022
8023 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8024 may alter controls if we determine we don't have to swap EFER after all. */
8025 rc = hmR0VmxExportGuestMsrs(pVCpu);
8026 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8027
8028 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8029 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8030
8031 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8032 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8033
8034 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8035 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8036 rc = hmR0VmxExportGuestRip(pVCpu);
8037 rc |= hmR0VmxExportGuestRsp(pVCpu);
8038 rc |= hmR0VmxExportGuestRflags(pVCpu);
8039 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8040
8041 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8042 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8043 | HM_CHANGED_GUEST_CR2
8044 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8045 | HM_CHANGED_GUEST_X87
8046 | HM_CHANGED_GUEST_SSE_AVX
8047 | HM_CHANGED_GUEST_OTHER_XSAVE
8048 | HM_CHANGED_GUEST_XCRx
8049 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8050 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8051 | HM_CHANGED_GUEST_TSC_AUX
8052 | HM_CHANGED_GUEST_OTHER_MSRS
8053 | HM_CHANGED_GUEST_HWVIRT
8054 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8055
8056 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8057 return rc;
8058}
8059
8060
8061/**
8062 * Exports the state shared between the host and guest into the VMCS.
8063 *
8064 * @param pVCpu The cross context virtual CPU structure.
8065 *
8066 * @remarks No-long-jump zone!!!
8067 */
8068static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8069{
8070 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8071 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8072
8073 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8074 {
8075 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8076 AssertRC(rc);
8077 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8078
8079 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8080 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8081 {
8082 rc = hmR0VmxExportGuestRflags(pVCpu);
8083 AssertRC(rc);
8084 }
8085 }
8086
8087 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8088 {
8089 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8090 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8091 }
8092
8093 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8094 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8095}
8096
8097
8098/**
8099 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8100 *
8101 * @returns Strict VBox status code (i.e. informational status codes too).
8102 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8103 * without unrestricted guest access and the VMMDev is not presently
8104 * mapped (e.g. EFI32).
8105 *
8106 * @param pVCpu The cross context virtual CPU structure.
8107 *
8108 * @remarks No-long-jump zone!!!
8109 */
8110static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8111{
8112 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8113 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8114 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8115
8116#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8117 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8118#endif
8119
8120 /*
8121 * For many exits it's only RIP that changes and hence try to export it first
8122 * without going through a lot of change flag checks.
8123 */
8124 VBOXSTRICTRC rcStrict;
8125 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8126 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8127 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8128 {
8129 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8130 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8131 { /* likely */}
8132 else
8133 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8134 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8135 }
8136 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8137 {
8138 rcStrict = hmR0VmxExportGuestState(pVCpu);
8139 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8140 { /* likely */}
8141 else
8142 {
8143 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8144 VBOXSTRICTRC_VAL(rcStrict)));
8145 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8146 return rcStrict;
8147 }
8148 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8149 }
8150 else
8151 rcStrict = VINF_SUCCESS;
8152
8153#ifdef VBOX_STRICT
8154 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8155 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8156 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8157 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8158 ("fCtxChanged=%#RX64\n", fCtxChanged));
8159#endif
8160 return rcStrict;
8161}
8162
8163
8164/**
8165 * Does the preparations before executing guest code in VT-x.
8166 *
8167 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8168 * recompiler/IEM. We must be cautious what we do here regarding committing
8169 * guest-state information into the VMCS assuming we assuredly execute the
8170 * guest in VT-x mode.
8171 *
8172 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8173 * the common-state (TRPM/forceflags), we must undo those changes so that the
8174 * recompiler/IEM can (and should) use them when it resumes guest execution.
8175 * Otherwise such operations must be done when we can no longer exit to ring-3.
8176 *
8177 * @returns Strict VBox status code (i.e. informational status codes too).
8178 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8179 * have been disabled.
8180 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8181 * double-fault into the guest.
8182 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8183 * dispatched directly.
8184 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8185 *
8186 * @param pVCpu The cross context virtual CPU structure.
8187 * @param pVmxTransient Pointer to the VMX transient structure.
8188 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8189 * us ignore some of the reasons for returning to
8190 * ring-3, and return VINF_EM_DBG_STEPPED if event
8191 * dispatching took place.
8192 */
8193static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8194{
8195 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8196
8197#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8198 PGMRZDynMapFlushAutoSet(pVCpu);
8199#endif
8200
8201 /* Check force flag actions that might require us to go back to ring-3. */
8202 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8203 if (rcStrict == VINF_SUCCESS)
8204 { /* FFs doesn't get set all the time. */ }
8205 else
8206 return rcStrict;
8207
8208 /*
8209 * Setup the virtualized-APIC accesses.
8210 *
8211 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8212 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8213 *
8214 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8215 */
8216 PVM pVM = pVCpu->CTX_SUFF(pVM);
8217 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8218 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8219 && PDMHasApic(pVM))
8220 {
8221 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8222 Assert(u64MsrApicBase);
8223 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8224
8225 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8226
8227 /* Unalias any existing mapping. */
8228 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8229 AssertRCReturn(rc, rc);
8230
8231 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8232 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8233 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8234 AssertRCReturn(rc, rc);
8235
8236 /* Update the per-VCPU cache of the APIC base MSR. */
8237 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8238 }
8239
8240 if (TRPMHasTrap(pVCpu))
8241 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8242 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8243
8244 /*
8245 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8246 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8247 * also result in triple-faulting the VM.
8248 */
8249 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8250 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8251 { /* likely */ }
8252 else
8253 {
8254 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8255 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8256 return rcStrict;
8257 }
8258
8259 /*
8260 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8261 * import CR3 themselves. We will need to update them here, as even as late as the above
8262 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8263 * the below force flags to be set.
8264 */
8265 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8266 {
8267 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8268 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8269 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8270 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8271 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8272 }
8273 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8274 {
8275 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8276 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8277 }
8278
8279 /*
8280 * No longjmps to ring-3 from this point on!!!
8281 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8282 * This also disables flushing of the R0-logger instance (if any).
8283 */
8284 VMMRZCallRing3Disable(pVCpu);
8285
8286 /*
8287 * Export the guest state bits.
8288 *
8289 * We cannot perform longjmps while loading the guest state because we do not preserve the
8290 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8291 * CPU migration.
8292 *
8293 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8294 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8295 * Hence, loading of the guest state needs to be done -after- injection of events.
8296 */
8297 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8298 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8299 { /* likely */ }
8300 else
8301 {
8302 VMMRZCallRing3Enable(pVCpu);
8303 return rcStrict;
8304 }
8305
8306 /*
8307 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8308 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8309 * preemption disabled for a while. Since this is purly to aid the
8310 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8311 * disable interrupt on NT.
8312 *
8313 * We need to check for force-flags that could've possible been altered since we last
8314 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8315 * see @bugref{6398}).
8316 *
8317 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8318 * to ring-3 before executing guest code.
8319 */
8320 pVmxTransient->fEFlags = ASMIntDisableFlags();
8321
8322 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8323 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8324 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8325 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8326 {
8327 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8328 {
8329 pVCpu->hm.s.Event.fPending = false;
8330
8331 /*
8332 * We've injected any pending events. This is really the point of no return (to ring-3).
8333 *
8334 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8335 * returns from this function, so don't enable them here.
8336 */
8337 return VINF_SUCCESS;
8338 }
8339
8340 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8341 rcStrict = VINF_EM_RAW_INTERRUPT;
8342 }
8343 else
8344 {
8345 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8346 rcStrict = VINF_EM_RAW_TO_R3;
8347 }
8348
8349 ASMSetFlags(pVmxTransient->fEFlags);
8350 VMMRZCallRing3Enable(pVCpu);
8351
8352 return rcStrict;
8353}
8354
8355
8356/**
8357 * Prepares to run guest code in VT-x and we've committed to doing so. This
8358 * means there is no backing out to ring-3 or anywhere else at this
8359 * point.
8360 *
8361 * @param pVCpu The cross context virtual CPU structure.
8362 * @param pVmxTransient Pointer to the VMX transient structure.
8363 *
8364 * @remarks Called with preemption disabled.
8365 * @remarks No-long-jump zone!!!
8366 */
8367static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8368{
8369 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8370 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8371 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8372
8373 /*
8374 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8375 */
8376 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8377 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8378
8379 PVM pVM = pVCpu->CTX_SUFF(pVM);
8380 if (!CPUMIsGuestFPUStateActive(pVCpu))
8381 {
8382 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8383 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8384 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8385 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8386 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8387 }
8388
8389 /*
8390 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8391 */
8392 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8393 && pVCpu->hm.s.vmx.cMsrs > 0)
8394 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8395
8396 /*
8397 * Re-save the host state bits as we may've been preempted (only happens when
8398 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8399 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8400 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8401 * See @bugref{8432}.
8402 */
8403 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8404 {
8405 int rc = hmR0VmxExportHostState(pVCpu);
8406 AssertRC(rc);
8407 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8408 }
8409 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8410
8411 /*
8412 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8413 */
8414 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8415 hmR0VmxExportSharedState(pVCpu);
8416 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8417
8418 /* Store status of the shared guest-host state at the time of VM-entry. */
8419#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8420 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8421 {
8422 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8423 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8424 }
8425 else
8426#endif
8427 {
8428 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8429 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8430 }
8431
8432 /*
8433 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8434 */
8435 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8436 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8437
8438 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8439 RTCPUID idCurrentCpu = pCpu->idCpu;
8440 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8441 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8442 {
8443 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8444 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8445 }
8446
8447 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8448 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8449 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8450 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8451
8452 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8453
8454 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8455 to start executing. */
8456
8457 /*
8458 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8459 */
8460 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8461 {
8462 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8463 {
8464 bool fMsrUpdated;
8465 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8466 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8467 &fMsrUpdated);
8468 AssertRC(rc2);
8469 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8470 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8471 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8472 }
8473 else
8474 {
8475 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8476 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8477 }
8478 }
8479
8480 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8481 {
8482 bool fMsrUpdated;
8483 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8484 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8485 &fMsrUpdated);
8486 AssertRC(rc2);
8487 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8488 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8489 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8490 }
8491
8492#ifdef VBOX_STRICT
8493 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8494 hmR0VmxCheckHostEferMsr(pVCpu);
8495 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8496#endif
8497#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8498 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8499 {
8500 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8501 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8502 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8503 }
8504#endif
8505}
8506
8507
8508/**
8509 * Performs some essential restoration of state after running guest code in
8510 * VT-x.
8511 *
8512 * @param pVCpu The cross context virtual CPU structure.
8513 * @param pVmxTransient Pointer to the VMX transient structure.
8514 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8515 *
8516 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8517 *
8518 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8519 * unconditionally when it is safe to do so.
8520 */
8521static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8522{
8523 uint64_t const uHostTsc = ASMReadTSC();
8524 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8525
8526 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8527 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8528 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8529 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8530 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8531 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8532
8533 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8534 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8535
8536 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8537 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8538 Assert(!ASMIntAreEnabled());
8539 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8540
8541#if HC_ARCH_BITS == 64
8542 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8543#endif
8544#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8545 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8546 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8547 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8548#else
8549 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8550#endif
8551#ifdef VBOX_STRICT
8552 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8553#endif
8554 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8555
8556 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8557 uint32_t uExitReason;
8558 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8559 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8560 AssertRC(rc);
8561 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8562 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8563
8564 if (rcVMRun == VINF_SUCCESS)
8565 {
8566 /*
8567 * Update the VM-exit history array here even if the VM-entry failed due to:
8568 * - Invalid guest state.
8569 * - MSR loading.
8570 * - Machine-check event.
8571 *
8572 * In any of the above cases we will still have a "valid" VM-exit reason
8573 * despite @a fVMEntryFailed being false.
8574 *
8575 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8576 *
8577 * Note! We don't have CS or RIP at this point. Will probably address that later
8578 * by amending the history entry added here.
8579 */
8580 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8581 UINT64_MAX, uHostTsc);
8582
8583 if (!pVmxTransient->fVMEntryFailed)
8584 {
8585 VMMRZCallRing3Enable(pVCpu);
8586
8587 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8588 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8589
8590#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8591 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8592 AssertRC(rc);
8593#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8594 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8595 AssertRC(rc);
8596#else
8597 /*
8598 * Import the guest-interruptibility state always as we need it while evaluating
8599 * injecting events on re-entry.
8600 *
8601 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8602 * checking for real-mode while exporting the state because all bits that cause
8603 * mode changes wrt CR0 are intercepted.
8604 */
8605 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8606 AssertRC(rc);
8607#endif
8608
8609 /*
8610 * Sync the TPR shadow with our APIC state.
8611 */
8612 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8613 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8614 {
8615 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8616 AssertRC(rc);
8617 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8618 }
8619
8620 return;
8621 }
8622 }
8623 else
8624 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8625
8626 VMMRZCallRing3Enable(pVCpu);
8627}
8628
8629
8630/**
8631 * Runs the guest code using VT-x the normal way.
8632 *
8633 * @returns VBox status code.
8634 * @param pVCpu The cross context virtual CPU structure.
8635 *
8636 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8637 */
8638static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8639{
8640 VMXTRANSIENT VmxTransient;
8641 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8642 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8643 uint32_t cLoops = 0;
8644
8645 for (;; cLoops++)
8646 {
8647 Assert(!HMR0SuspendPending());
8648 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8649
8650 /* Preparatory work for running guest code, this may force us to return
8651 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8652 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8653 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8654 if (rcStrict != VINF_SUCCESS)
8655 break;
8656
8657 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8658 int rcRun = hmR0VmxRunGuest(pVCpu);
8659
8660 /* Restore any residual host-state and save any bits shared between host
8661 and guest into the guest-CPU state. Re-enables interrupts! */
8662 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8663
8664 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8665 if (RT_SUCCESS(rcRun))
8666 { /* very likely */ }
8667 else
8668 {
8669 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8670 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8671 return rcRun;
8672 }
8673
8674 /* Profile the VM-exit. */
8675 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8676 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8677 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8678 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8679 HMVMX_START_EXIT_DISPATCH_PROF();
8680
8681 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8682
8683 /* Handle the VM-exit. */
8684#ifdef HMVMX_USE_FUNCTION_TABLE
8685 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8686#else
8687 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8688#endif
8689 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8690 if (rcStrict == VINF_SUCCESS)
8691 {
8692 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8693 continue; /* likely */
8694 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8695 rcStrict = VINF_EM_RAW_INTERRUPT;
8696 }
8697 break;
8698 }
8699
8700 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8701 return rcStrict;
8702}
8703
8704
8705
8706/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8707 * probes.
8708 *
8709 * The following few functions and associated structure contains the bloat
8710 * necessary for providing detailed debug events and dtrace probes as well as
8711 * reliable host side single stepping. This works on the principle of
8712 * "subclassing" the normal execution loop and workers. We replace the loop
8713 * method completely and override selected helpers to add necessary adjustments
8714 * to their core operation.
8715 *
8716 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8717 * any performance for debug and analysis features.
8718 *
8719 * @{
8720 */
8721
8722/**
8723 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8724 * the debug run loop.
8725 */
8726typedef struct VMXRUNDBGSTATE
8727{
8728 /** The RIP we started executing at. This is for detecting that we stepped. */
8729 uint64_t uRipStart;
8730 /** The CS we started executing with. */
8731 uint16_t uCsStart;
8732
8733 /** Whether we've actually modified the 1st execution control field. */
8734 bool fModifiedProcCtls : 1;
8735 /** Whether we've actually modified the 2nd execution control field. */
8736 bool fModifiedProcCtls2 : 1;
8737 /** Whether we've actually modified the exception bitmap. */
8738 bool fModifiedXcptBitmap : 1;
8739
8740 /** We desire the modified the CR0 mask to be cleared. */
8741 bool fClearCr0Mask : 1;
8742 /** We desire the modified the CR4 mask to be cleared. */
8743 bool fClearCr4Mask : 1;
8744 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8745 uint32_t fCpe1Extra;
8746 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8747 uint32_t fCpe1Unwanted;
8748 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8749 uint32_t fCpe2Extra;
8750 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8751 uint32_t bmXcptExtra;
8752 /** The sequence number of the Dtrace provider settings the state was
8753 * configured against. */
8754 uint32_t uDtraceSettingsSeqNo;
8755 /** VM-exits to check (one bit per VM-exit). */
8756 uint32_t bmExitsToCheck[3];
8757
8758 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8759 uint32_t fProcCtlsInitial;
8760 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8761 uint32_t fProcCtls2Initial;
8762 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8763 uint32_t bmXcptInitial;
8764} VMXRUNDBGSTATE;
8765AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8766typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8767
8768
8769/**
8770 * Initializes the VMXRUNDBGSTATE structure.
8771 *
8772 * @param pVCpu The cross context virtual CPU structure of the
8773 * calling EMT.
8774 * @param pDbgState The structure to initialize.
8775 */
8776static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8777{
8778 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8779 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8780
8781 pDbgState->fModifiedProcCtls = false;
8782 pDbgState->fModifiedProcCtls2 = false;
8783 pDbgState->fModifiedXcptBitmap = false;
8784 pDbgState->fClearCr0Mask = false;
8785 pDbgState->fClearCr4Mask = false;
8786 pDbgState->fCpe1Extra = 0;
8787 pDbgState->fCpe1Unwanted = 0;
8788 pDbgState->fCpe2Extra = 0;
8789 pDbgState->bmXcptExtra = 0;
8790 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8791 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8792 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8793}
8794
8795
8796/**
8797 * Updates the VMSC fields with changes requested by @a pDbgState.
8798 *
8799 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8800 * immediately before executing guest code, i.e. when interrupts are disabled.
8801 * We don't check status codes here as we cannot easily assert or return in the
8802 * latter case.
8803 *
8804 * @param pVCpu The cross context virtual CPU structure.
8805 * @param pDbgState The debug state.
8806 */
8807static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8808{
8809 /*
8810 * Ensure desired flags in VMCS control fields are set.
8811 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8812 *
8813 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8814 * there should be no stale data in pCtx at this point.
8815 */
8816 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8817 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8818 {
8819 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8820 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8821 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8822 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8823 pDbgState->fModifiedProcCtls = true;
8824 }
8825
8826 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8827 {
8828 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8829 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8830 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8831 pDbgState->fModifiedProcCtls2 = true;
8832 }
8833
8834 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8835 {
8836 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8837 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8838 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8839 pDbgState->fModifiedXcptBitmap = true;
8840 }
8841
8842 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8843 {
8844 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8845 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8846 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8847 }
8848
8849 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8850 {
8851 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8852 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8853 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8854 }
8855}
8856
8857
8858/**
8859 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
8860 * re-entry next time around.
8861 *
8862 * @returns Strict VBox status code (i.e. informational status codes too).
8863 * @param pVCpu The cross context virtual CPU structure.
8864 * @param pDbgState The debug state.
8865 * @param rcStrict The return code from executing the guest using single
8866 * stepping.
8867 */
8868static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8869{
8870 /*
8871 * Restore VM-exit control settings as we may not reenter this function the
8872 * next time around.
8873 */
8874 /* We reload the initial value, trigger what we can of recalculations the
8875 next time around. From the looks of things, that's all that's required atm. */
8876 if (pDbgState->fModifiedProcCtls)
8877 {
8878 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8879 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8880 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8881 AssertRCReturn(rc2, rc2);
8882 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8883 }
8884
8885 /* We're currently the only ones messing with this one, so just restore the
8886 cached value and reload the field. */
8887 if ( pDbgState->fModifiedProcCtls2
8888 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8889 {
8890 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8891 AssertRCReturn(rc2, rc2);
8892 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8893 }
8894
8895 /* If we've modified the exception bitmap, we restore it and trigger
8896 reloading and partial recalculation the next time around. */
8897 if (pDbgState->fModifiedXcptBitmap)
8898 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8899
8900 return rcStrict;
8901}
8902
8903
8904/**
8905 * Configures VM-exit controls for current DBGF and DTrace settings.
8906 *
8907 * This updates @a pDbgState and the VMCS execution control fields to reflect
8908 * the necessary VM-exits demanded by DBGF and DTrace.
8909 *
8910 * @param pVCpu The cross context virtual CPU structure.
8911 * @param pDbgState The debug state.
8912 * @param pVmxTransient Pointer to the VMX transient structure. May update
8913 * fUpdateTscOffsettingAndPreemptTimer.
8914 */
8915static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8916{
8917 /*
8918 * Take down the dtrace serial number so we can spot changes.
8919 */
8920 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8921 ASMCompilerBarrier();
8922
8923 /*
8924 * We'll rebuild most of the middle block of data members (holding the
8925 * current settings) as we go along here, so start by clearing it all.
8926 */
8927 pDbgState->bmXcptExtra = 0;
8928 pDbgState->fCpe1Extra = 0;
8929 pDbgState->fCpe1Unwanted = 0;
8930 pDbgState->fCpe2Extra = 0;
8931 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8932 pDbgState->bmExitsToCheck[i] = 0;
8933
8934 /*
8935 * Software interrupts (INT XXh) - no idea how to trigger these...
8936 */
8937 PVM pVM = pVCpu->CTX_SUFF(pVM);
8938 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8939 || VBOXVMM_INT_SOFTWARE_ENABLED())
8940 {
8941 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8942 }
8943
8944 /*
8945 * INT3 breakpoints - triggered by #BP exceptions.
8946 */
8947 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8948 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8949
8950 /*
8951 * Exception bitmap and XCPT events+probes.
8952 */
8953 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8954 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8955 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8956
8957 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8958 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8959 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8960 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8961 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8962 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8963 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8964 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8965 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8966 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8967 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8968 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8969 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8970 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8971 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8972 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8973 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8974 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8975
8976 if (pDbgState->bmXcptExtra)
8977 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8978
8979 /*
8980 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8981 *
8982 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8983 * So, when adding/changing/removing please don't forget to update it.
8984 *
8985 * Some of the macros are picking up local variables to save horizontal space,
8986 * (being able to see it in a table is the lesser evil here).
8987 */
8988#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8989 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8990 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8991#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8992 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8993 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8994 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8995 } else do { } while (0)
8996#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8997 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8998 { \
8999 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9000 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9001 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9002 } else do { } while (0)
9003#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9004 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9005 { \
9006 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9007 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9008 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9009 } else do { } while (0)
9010#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9011 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9012 { \
9013 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9014 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9015 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9016 } else do { } while (0)
9017
9018 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9019 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9020 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9021 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9022 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9023
9024 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9025 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9026 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9027 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9028 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9029 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9030 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9031 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9032 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9033 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9034 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9035 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9036 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9037 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9038 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9039 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9040 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9044 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9048 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9050 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9052 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9054 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9056 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9057 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9058 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9059 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9060
9061 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9062 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9063 {
9064 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9065 AssertRC(rc);
9066
9067#if 0 /** @todo fix me */
9068 pDbgState->fClearCr0Mask = true;
9069 pDbgState->fClearCr4Mask = true;
9070#endif
9071 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9072 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9073 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9074 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9075 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9076 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9077 require clearing here and in the loop if we start using it. */
9078 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9079 }
9080 else
9081 {
9082 if (pDbgState->fClearCr0Mask)
9083 {
9084 pDbgState->fClearCr0Mask = false;
9085 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9086 }
9087 if (pDbgState->fClearCr4Mask)
9088 {
9089 pDbgState->fClearCr4Mask = false;
9090 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9091 }
9092 }
9093 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9094 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9095
9096 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9097 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9098 {
9099 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9100 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9101 }
9102 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9104
9105 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9106 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9107 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9108 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9109 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9110 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9111 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9112 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9113#if 0 /** @todo too slow, fix handler. */
9114 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9115#endif
9116 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9117
9118 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9119 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9120 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9121 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9122 {
9123 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9124 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9125 }
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9127 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9128 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9129 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9130
9131 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9132 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9133 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9134 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9135 {
9136 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9137 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9138 }
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9141 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9143
9144 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9146 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9148 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9150 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9152 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9154 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9156 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9158 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9160 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9161 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9162 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9163 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9164 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9166
9167#undef IS_EITHER_ENABLED
9168#undef SET_ONLY_XBM_IF_EITHER_EN
9169#undef SET_CPE1_XBM_IF_EITHER_EN
9170#undef SET_CPEU_XBM_IF_EITHER_EN
9171#undef SET_CPE2_XBM_IF_EITHER_EN
9172
9173 /*
9174 * Sanitize the control stuff.
9175 */
9176 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9177 if (pDbgState->fCpe2Extra)
9178 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9179 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9180 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9181 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9182 {
9183 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9184 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9185 }
9186
9187 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9188 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9189 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9190 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9191}
9192
9193
9194/**
9195 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9196 * appropriate.
9197 *
9198 * The caller has checked the VM-exit against the
9199 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9200 * already, so we don't have to do that either.
9201 *
9202 * @returns Strict VBox status code (i.e. informational status codes too).
9203 * @param pVCpu The cross context virtual CPU structure.
9204 * @param pVmxTransient Pointer to the VMX-transient structure.
9205 * @param uExitReason The VM-exit reason.
9206 *
9207 * @remarks The name of this function is displayed by dtrace, so keep it short
9208 * and to the point. No longer than 33 chars long, please.
9209 */
9210static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9211{
9212 /*
9213 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9214 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9215 *
9216 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9217 * does. Must add/change/remove both places. Same ordering, please.
9218 *
9219 * Added/removed events must also be reflected in the next section
9220 * where we dispatch dtrace events.
9221 */
9222 bool fDtrace1 = false;
9223 bool fDtrace2 = false;
9224 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9225 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9226 uint32_t uEventArg = 0;
9227#define SET_EXIT(a_EventSubName) \
9228 do { \
9229 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9230 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9231 } while (0)
9232#define SET_BOTH(a_EventSubName) \
9233 do { \
9234 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9235 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9236 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9237 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9238 } while (0)
9239 switch (uExitReason)
9240 {
9241 case VMX_EXIT_MTF:
9242 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9243
9244 case VMX_EXIT_XCPT_OR_NMI:
9245 {
9246 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9247 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9248 {
9249 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9250 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9251 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9252 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9253 {
9254 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9255 {
9256 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9257 uEventArg = pVmxTransient->uExitIntErrorCode;
9258 }
9259 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9260 switch (enmEvent1)
9261 {
9262 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9263 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9264 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9265 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9266 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9267 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9268 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9269 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9270 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9271 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9272 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9273 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9274 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9275 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9276 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9277 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9278 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9279 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9280 default: break;
9281 }
9282 }
9283 else
9284 AssertFailed();
9285 break;
9286
9287 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9288 uEventArg = idxVector;
9289 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9290 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9291 break;
9292 }
9293 break;
9294 }
9295
9296 case VMX_EXIT_TRIPLE_FAULT:
9297 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9298 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9299 break;
9300 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9301 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9302 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9303 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9304 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9305
9306 /* Instruction specific VM-exits: */
9307 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9308 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9309 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9310 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9311 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9312 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9313 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9314 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9315 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9316 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9317 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9318 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9319 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9320 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9321 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9322 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9323 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9324 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9325 case VMX_EXIT_MOV_CRX:
9326 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9327 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9328 SET_BOTH(CRX_READ);
9329 else
9330 SET_BOTH(CRX_WRITE);
9331 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9332 break;
9333 case VMX_EXIT_MOV_DRX:
9334 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9335 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9336 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9337 SET_BOTH(DRX_READ);
9338 else
9339 SET_BOTH(DRX_WRITE);
9340 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9341 break;
9342 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9343 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9344 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9345 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9346 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9347 case VMX_EXIT_XDTR_ACCESS:
9348 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9349 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9350 {
9351 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9352 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9353 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9354 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9355 }
9356 break;
9357
9358 case VMX_EXIT_TR_ACCESS:
9359 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9360 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9361 {
9362 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9363 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9364 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9365 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9366 }
9367 break;
9368
9369 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9370 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9371 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9372 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9373 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9374 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9375 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9376 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9377 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9378 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9379 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9380
9381 /* Events that aren't relevant at this point. */
9382 case VMX_EXIT_EXT_INT:
9383 case VMX_EXIT_INT_WINDOW:
9384 case VMX_EXIT_NMI_WINDOW:
9385 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9386 case VMX_EXIT_PREEMPT_TIMER:
9387 case VMX_EXIT_IO_INSTR:
9388 break;
9389
9390 /* Errors and unexpected events. */
9391 case VMX_EXIT_INIT_SIGNAL:
9392 case VMX_EXIT_SIPI:
9393 case VMX_EXIT_IO_SMI:
9394 case VMX_EXIT_SMI:
9395 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9396 case VMX_EXIT_ERR_MSR_LOAD:
9397 case VMX_EXIT_ERR_MACHINE_CHECK:
9398 break;
9399
9400 default:
9401 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9402 break;
9403 }
9404#undef SET_BOTH
9405#undef SET_EXIT
9406
9407 /*
9408 * Dtrace tracepoints go first. We do them here at once so we don't
9409 * have to copy the guest state saving and stuff a few dozen times.
9410 * Down side is that we've got to repeat the switch, though this time
9411 * we use enmEvent since the probes are a subset of what DBGF does.
9412 */
9413 if (fDtrace1 || fDtrace2)
9414 {
9415 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9416 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9417 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9418 switch (enmEvent1)
9419 {
9420 /** @todo consider which extra parameters would be helpful for each probe. */
9421 case DBGFEVENT_END: break;
9422 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9424 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9426 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9427 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9428 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9429 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9430 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9431 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9432 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9433 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9434 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9435 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9436 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9437 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9438 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9439 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9440 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9441 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9442 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9445 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9446 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9447 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9448 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9449 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9450 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9451 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9452 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9453 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9454 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9455 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9456 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9483 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9484 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9485 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9486 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9487 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9488 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9489 }
9490 switch (enmEvent2)
9491 {
9492 /** @todo consider which extra parameters would be helpful for each probe. */
9493 case DBGFEVENT_END: break;
9494 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9496 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9499 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9500 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9501 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9502 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9503 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9504 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9505 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9506 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9507 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9508 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9509 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9510 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9541 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9542 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9543 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9544 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9545 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9546 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9547 }
9548 }
9549
9550 /*
9551 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9552 * the DBGF call will do a full check).
9553 *
9554 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9555 * Note! If we have to events, we prioritize the first, i.e. the instruction
9556 * one, in order to avoid event nesting.
9557 */
9558 PVM pVM = pVCpu->CTX_SUFF(pVM);
9559 if ( enmEvent1 != DBGFEVENT_END
9560 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9561 {
9562 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9563 if (rcStrict != VINF_SUCCESS)
9564 return rcStrict;
9565 }
9566 else if ( enmEvent2 != DBGFEVENT_END
9567 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9568 {
9569 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9570 if (rcStrict != VINF_SUCCESS)
9571 return rcStrict;
9572 }
9573
9574 return VINF_SUCCESS;
9575}
9576
9577
9578/**
9579 * Single-stepping VM-exit filtering.
9580 *
9581 * This is preprocessing the VM-exits and deciding whether we've gotten far
9582 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9583 * handling is performed.
9584 *
9585 * @returns Strict VBox status code (i.e. informational status codes too).
9586 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9587 * @param pVmxTransient Pointer to the VMX-transient structure.
9588 * @param pDbgState The debug state.
9589 */
9590DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9591{
9592 /*
9593 * Expensive (saves context) generic dtrace VM-exit probe.
9594 */
9595 uint32_t const uExitReason = pVmxTransient->uExitReason;
9596 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9597 { /* more likely */ }
9598 else
9599 {
9600 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9601 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9602 AssertRC(rc);
9603 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9604 }
9605
9606 /*
9607 * Check for host NMI, just to get that out of the way.
9608 */
9609 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9610 { /* normally likely */ }
9611 else
9612 {
9613 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9614 AssertRCReturn(rc2, rc2);
9615 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9616 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9617 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9618 }
9619
9620 /*
9621 * Check for single stepping event if we're stepping.
9622 */
9623 if (pVCpu->hm.s.fSingleInstruction)
9624 {
9625 switch (uExitReason)
9626 {
9627 case VMX_EXIT_MTF:
9628 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9629
9630 /* Various events: */
9631 case VMX_EXIT_XCPT_OR_NMI:
9632 case VMX_EXIT_EXT_INT:
9633 case VMX_EXIT_TRIPLE_FAULT:
9634 case VMX_EXIT_INT_WINDOW:
9635 case VMX_EXIT_NMI_WINDOW:
9636 case VMX_EXIT_TASK_SWITCH:
9637 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9638 case VMX_EXIT_APIC_ACCESS:
9639 case VMX_EXIT_EPT_VIOLATION:
9640 case VMX_EXIT_EPT_MISCONFIG:
9641 case VMX_EXIT_PREEMPT_TIMER:
9642
9643 /* Instruction specific VM-exits: */
9644 case VMX_EXIT_CPUID:
9645 case VMX_EXIT_GETSEC:
9646 case VMX_EXIT_HLT:
9647 case VMX_EXIT_INVD:
9648 case VMX_EXIT_INVLPG:
9649 case VMX_EXIT_RDPMC:
9650 case VMX_EXIT_RDTSC:
9651 case VMX_EXIT_RSM:
9652 case VMX_EXIT_VMCALL:
9653 case VMX_EXIT_VMCLEAR:
9654 case VMX_EXIT_VMLAUNCH:
9655 case VMX_EXIT_VMPTRLD:
9656 case VMX_EXIT_VMPTRST:
9657 case VMX_EXIT_VMREAD:
9658 case VMX_EXIT_VMRESUME:
9659 case VMX_EXIT_VMWRITE:
9660 case VMX_EXIT_VMXOFF:
9661 case VMX_EXIT_VMXON:
9662 case VMX_EXIT_MOV_CRX:
9663 case VMX_EXIT_MOV_DRX:
9664 case VMX_EXIT_IO_INSTR:
9665 case VMX_EXIT_RDMSR:
9666 case VMX_EXIT_WRMSR:
9667 case VMX_EXIT_MWAIT:
9668 case VMX_EXIT_MONITOR:
9669 case VMX_EXIT_PAUSE:
9670 case VMX_EXIT_XDTR_ACCESS:
9671 case VMX_EXIT_TR_ACCESS:
9672 case VMX_EXIT_INVEPT:
9673 case VMX_EXIT_RDTSCP:
9674 case VMX_EXIT_INVVPID:
9675 case VMX_EXIT_WBINVD:
9676 case VMX_EXIT_XSETBV:
9677 case VMX_EXIT_RDRAND:
9678 case VMX_EXIT_INVPCID:
9679 case VMX_EXIT_VMFUNC:
9680 case VMX_EXIT_RDSEED:
9681 case VMX_EXIT_XSAVES:
9682 case VMX_EXIT_XRSTORS:
9683 {
9684 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9685 AssertRCReturn(rc, rc);
9686 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9687 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9688 return VINF_EM_DBG_STEPPED;
9689 break;
9690 }
9691
9692 /* Errors and unexpected events: */
9693 case VMX_EXIT_INIT_SIGNAL:
9694 case VMX_EXIT_SIPI:
9695 case VMX_EXIT_IO_SMI:
9696 case VMX_EXIT_SMI:
9697 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9698 case VMX_EXIT_ERR_MSR_LOAD:
9699 case VMX_EXIT_ERR_MACHINE_CHECK:
9700 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9701 break;
9702
9703 default:
9704 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9705 break;
9706 }
9707 }
9708
9709 /*
9710 * Check for debugger event breakpoints and dtrace probes.
9711 */
9712 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9713 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9714 {
9715 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9716 if (rcStrict != VINF_SUCCESS)
9717 return rcStrict;
9718 }
9719
9720 /*
9721 * Normal processing.
9722 */
9723#ifdef HMVMX_USE_FUNCTION_TABLE
9724 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9725#else
9726 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9727#endif
9728}
9729
9730
9731/**
9732 * Single steps guest code using VT-x.
9733 *
9734 * @returns Strict VBox status code (i.e. informational status codes too).
9735 * @param pVCpu The cross context virtual CPU structure.
9736 *
9737 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9738 */
9739static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9740{
9741 VMXTRANSIENT VmxTransient;
9742 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9743
9744 /* Set HMCPU indicators. */
9745 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9746 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9747 pVCpu->hm.s.fDebugWantRdTscExit = false;
9748 pVCpu->hm.s.fUsingDebugLoop = true;
9749
9750 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9751 VMXRUNDBGSTATE DbgState;
9752 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9753 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9754
9755 /*
9756 * The loop.
9757 */
9758 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9759 for (uint32_t cLoops = 0; ; cLoops++)
9760 {
9761 Assert(!HMR0SuspendPending());
9762 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9763 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9764
9765 /*
9766 * Preparatory work for running guest code, this may force us to return
9767 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9768 */
9769 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9770 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9771 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9772 if (rcStrict != VINF_SUCCESS)
9773 break;
9774
9775 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9776 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9777
9778 /*
9779 * Now we can run the guest code.
9780 */
9781 int rcRun = hmR0VmxRunGuest(pVCpu);
9782
9783 /*
9784 * Restore any residual host-state and save any bits shared between host
9785 * and guest into the guest-CPU state. Re-enables interrupts!
9786 */
9787 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9788
9789 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9790 if (RT_SUCCESS(rcRun))
9791 { /* very likely */ }
9792 else
9793 {
9794 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9795 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9796 return rcRun;
9797 }
9798
9799 /* Profile the VM-exit. */
9800 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9801 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9802 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9803 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9804 HMVMX_START_EXIT_DISPATCH_PROF();
9805
9806 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9807
9808 /*
9809 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9810 */
9811 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9812 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9813 if (rcStrict != VINF_SUCCESS)
9814 break;
9815 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9816 {
9817 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9818 rcStrict = VINF_EM_RAW_INTERRUPT;
9819 break;
9820 }
9821
9822 /*
9823 * Stepping: Did the RIP change, if so, consider it a single step.
9824 * Otherwise, make sure one of the TFs gets set.
9825 */
9826 if (fStepping)
9827 {
9828 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9829 AssertRC(rc);
9830 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9831 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9832 {
9833 rcStrict = VINF_EM_DBG_STEPPED;
9834 break;
9835 }
9836 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9837 }
9838
9839 /*
9840 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9841 */
9842 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9843 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9844 }
9845
9846 /*
9847 * Clear the X86_EFL_TF if necessary.
9848 */
9849 if (pVCpu->hm.s.fClearTrapFlag)
9850 {
9851 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9852 AssertRC(rc);
9853 pVCpu->hm.s.fClearTrapFlag = false;
9854 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9855 }
9856 /** @todo there seems to be issues with the resume flag when the monitor trap
9857 * flag is pending without being used. Seen early in bios init when
9858 * accessing APIC page in protected mode. */
9859
9860 /*
9861 * Restore VM-exit control settings as we may not reenter this function the
9862 * next time around.
9863 */
9864 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9865
9866 /* Restore HMCPU indicators. */
9867 pVCpu->hm.s.fUsingDebugLoop = false;
9868 pVCpu->hm.s.fDebugWantRdTscExit = false;
9869 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9870
9871 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9872 return rcStrict;
9873}
9874
9875
9876/** @} */
9877
9878
9879/**
9880 * Checks if any expensive dtrace probes are enabled and we should go to the
9881 * debug loop.
9882 *
9883 * @returns true if we should use debug loop, false if not.
9884 */
9885static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9886{
9887 /* It's probably faster to OR the raw 32-bit counter variables together.
9888 Since the variables are in an array and the probes are next to one
9889 another (more or less), we have good locality. So, better read
9890 eight-nine cache lines ever time and only have one conditional, than
9891 128+ conditionals, right? */
9892 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9893 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9894 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9895 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9896 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9897 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9898 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9899 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9900 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9901 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9902 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9903 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9904 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9905 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9906 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9907 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9908 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9909 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9910 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9911 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9912 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9913 ) != 0
9914 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9915 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9916 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9917 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9918 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9919 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9920 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9921 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9922 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9923 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9924 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9925 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9926 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9927 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9928 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9929 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9930 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9931 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9932 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9935 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9936 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9937 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9938 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9939 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9940 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9941 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9942 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9943 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9944 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9945 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9956 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9957 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9958 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9959 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9960 ) != 0
9961 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9962 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9963 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9964 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9965 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9966 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9967 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9968 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9969 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9970 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9971 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9972 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9973 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9974 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9975 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9976 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9977 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9978 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9979 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9980 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9983 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9984 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9985 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9986 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9987 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9988 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9989 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9990 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9991 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9992 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9993 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10008 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10009 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10010 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10011 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10012 ) != 0;
10013}
10014
10015
10016/**
10017 * Runs the guest code using VT-x.
10018 *
10019 * @returns Strict VBox status code (i.e. informational status codes too).
10020 * @param pVCpu The cross context virtual CPU structure.
10021 */
10022VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10023{
10024 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10025 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10026 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10027 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10028
10029 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10030
10031 VBOXSTRICTRC rcStrict;
10032 if ( !pVCpu->hm.s.fUseDebugLoop
10033 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10034 && !DBGFIsStepping(pVCpu)
10035 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10036 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10037 else
10038 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10039
10040 if (rcStrict == VERR_EM_INTERPRETER)
10041 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10042 else if (rcStrict == VINF_EM_RESET)
10043 rcStrict = VINF_EM_TRIPLE_FAULT;
10044
10045 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10046 if (RT_FAILURE(rc2))
10047 {
10048 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10049 rcStrict = rc2;
10050 }
10051 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10052 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10053 return rcStrict;
10054}
10055
10056
10057#ifndef HMVMX_USE_FUNCTION_TABLE
10058DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10059{
10060#ifdef DEBUG_ramshankar
10061#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10062 do { \
10063 if (a_fSave != 0) \
10064 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10065 VBOXSTRICTRC rcStrict = a_CallExpr; \
10066 if (a_fSave != 0) \
10067 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10068 return rcStrict; \
10069 } while (0)
10070#else
10071# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10072#endif
10073 switch (rcReason)
10074 {
10075 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10076 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10077 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10078 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10079 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10080 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10081 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10082 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10083 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10084 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10085 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10086 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10087 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10088 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10089 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10090 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10091 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10092 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10093 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10094 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10095 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10096 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10097 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10098 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10099 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10100 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10101 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10102 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10103 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10104 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10105 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10106 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10107 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10108 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10109
10110 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10111 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10112 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10113 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10114 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10115 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10116 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10117 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10118 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10119
10120 case VMX_EXIT_VMCLEAR:
10121 case VMX_EXIT_VMLAUNCH:
10122 case VMX_EXIT_VMPTRLD:
10123 case VMX_EXIT_VMPTRST:
10124 case VMX_EXIT_VMREAD:
10125 case VMX_EXIT_VMRESUME:
10126 case VMX_EXIT_VMWRITE:
10127 case VMX_EXIT_VMXOFF:
10128 case VMX_EXIT_VMXON:
10129 case VMX_EXIT_INVEPT:
10130 case VMX_EXIT_INVVPID:
10131 case VMX_EXIT_VMFUNC:
10132 case VMX_EXIT_XSAVES:
10133 case VMX_EXIT_XRSTORS:
10134 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10135
10136 case VMX_EXIT_ENCLS:
10137 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10138 case VMX_EXIT_PML_FULL:
10139 default:
10140 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10141 }
10142#undef VMEXIT_CALL_RET
10143}
10144#endif /* !HMVMX_USE_FUNCTION_TABLE */
10145
10146
10147#ifdef VBOX_STRICT
10148/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10149# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10150 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10151
10152# define HMVMX_ASSERT_PREEMPT_CPUID() \
10153 do { \
10154 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10155 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10156 } while (0)
10157
10158# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10159 do { \
10160 AssertPtr((a_pVCpu)); \
10161 AssertPtr((a_pVmxTransient)); \
10162 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10163 Assert(ASMIntAreEnabled()); \
10164 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10165 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10166 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10167 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10168 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10169 HMVMX_ASSERT_PREEMPT_CPUID(); \
10170 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10171 } while (0)
10172
10173# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10174 do { \
10175 Log4Func(("\n")); \
10176 } while (0)
10177#else
10178# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10179 do { \
10180 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10181 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10182 } while (0)
10183# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10184#endif
10185
10186
10187/**
10188 * Advances the guest RIP by the specified number of bytes.
10189 *
10190 * @param pVCpu The cross context virtual CPU structure.
10191 * @param cbInstr Number of bytes to advance the RIP by.
10192 *
10193 * @remarks No-long-jump zone!!!
10194 */
10195DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10196{
10197 /* Advance the RIP. */
10198 pVCpu->cpum.GstCtx.rip += cbInstr;
10199 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10200
10201 /* Update interrupt inhibition. */
10202 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10203 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10204 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10205}
10206
10207
10208/**
10209 * Advances the guest RIP after reading it from the VMCS.
10210 *
10211 * @returns VBox status code, no informational status codes.
10212 * @param pVCpu The cross context virtual CPU structure.
10213 * @param pVmxTransient Pointer to the VMX transient structure.
10214 *
10215 * @remarks No-long-jump zone!!!
10216 */
10217static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10218{
10219 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10220 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10221 AssertRCReturn(rc, rc);
10222
10223 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10224
10225 /*
10226 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10227 * pending debug exception field as it takes care of priority of events.
10228 *
10229 * See Intel spec. 32.2.1 "Debug Exceptions".
10230 */
10231 if ( !pVCpu->hm.s.fSingleInstruction
10232 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10233 {
10234 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10235 AssertRCReturn(rc, rc);
10236 }
10237
10238 return VINF_SUCCESS;
10239}
10240
10241
10242/**
10243 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10244 * and update error record fields accordingly.
10245 *
10246 * @return VMX_IGS_* return codes.
10247 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10248 * wrong with the guest state.
10249 *
10250 * @param pVCpu The cross context virtual CPU structure.
10251 *
10252 * @remarks This function assumes our cache of the VMCS controls
10253 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10254 */
10255static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10256{
10257#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10258#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10259 uError = (err); \
10260 break; \
10261 } else do { } while (0)
10262
10263 int rc;
10264 PVM pVM = pVCpu->CTX_SUFF(pVM);
10265 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10266 uint32_t uError = VMX_IGS_ERROR;
10267 uint32_t u32Val;
10268 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10269
10270 do
10271 {
10272 /*
10273 * CR0.
10274 */
10275 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10276 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10277 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10278 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10279 if (fUnrestrictedGuest)
10280 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10281
10282 uint32_t u32GuestCr0;
10283 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10284 AssertRCBreak(rc);
10285 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10286 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10287 if ( !fUnrestrictedGuest
10288 && (u32GuestCr0 & X86_CR0_PG)
10289 && !(u32GuestCr0 & X86_CR0_PE))
10290 {
10291 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10292 }
10293
10294 /*
10295 * CR4.
10296 */
10297 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10298 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10299
10300 uint32_t u32GuestCr4;
10301 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10302 AssertRCBreak(rc);
10303 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10304 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10305
10306 /*
10307 * IA32_DEBUGCTL MSR.
10308 */
10309 uint64_t u64Val;
10310 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10311 AssertRCBreak(rc);
10312 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10313 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10314 {
10315 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10316 }
10317 uint64_t u64DebugCtlMsr = u64Val;
10318
10319#ifdef VBOX_STRICT
10320 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10321 AssertRCBreak(rc);
10322 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10323#endif
10324 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10325
10326 /*
10327 * RIP and RFLAGS.
10328 */
10329 uint32_t u32Eflags;
10330#if HC_ARCH_BITS == 64
10331 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10332 AssertRCBreak(rc);
10333 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10334 if ( !fLongModeGuest
10335 || !pCtx->cs.Attr.n.u1Long)
10336 {
10337 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10338 }
10339 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10340 * must be identical if the "IA-32e mode guest" VM-entry
10341 * control is 1 and CS.L is 1. No check applies if the
10342 * CPU supports 64 linear-address bits. */
10343
10344 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10345 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10346 AssertRCBreak(rc);
10347 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10348 VMX_IGS_RFLAGS_RESERVED);
10349 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10350 u32Eflags = u64Val;
10351#else
10352 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10353 AssertRCBreak(rc);
10354 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10355 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10356#endif
10357
10358 if ( fLongModeGuest
10359 || ( fUnrestrictedGuest
10360 && !(u32GuestCr0 & X86_CR0_PE)))
10361 {
10362 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10363 }
10364
10365 uint32_t u32EntryInfo;
10366 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10367 AssertRCBreak(rc);
10368 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10369 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10370 {
10371 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10372 }
10373
10374 /*
10375 * 64-bit checks.
10376 */
10377#if HC_ARCH_BITS == 64
10378 if (fLongModeGuest)
10379 {
10380 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10381 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10382 }
10383
10384 if ( !fLongModeGuest
10385 && (u32GuestCr4 & X86_CR4_PCIDE))
10386 {
10387 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10388 }
10389
10390 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10391 * 51:32 beyond the processor's physical-address width are 0. */
10392
10393 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10394 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10395 {
10396 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10397 }
10398
10399 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10400 AssertRCBreak(rc);
10401 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10402
10403 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10404 AssertRCBreak(rc);
10405 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10406#endif
10407
10408 /*
10409 * PERF_GLOBAL MSR.
10410 */
10411 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10412 {
10413 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10414 AssertRCBreak(rc);
10415 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10416 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10417 }
10418
10419 /*
10420 * PAT MSR.
10421 */
10422 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10423 {
10424 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10425 AssertRCBreak(rc);
10426 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10427 for (unsigned i = 0; i < 8; i++)
10428 {
10429 uint8_t u8Val = (u64Val & 0xff);
10430 if ( u8Val != 0 /* UC */
10431 && u8Val != 1 /* WC */
10432 && u8Val != 4 /* WT */
10433 && u8Val != 5 /* WP */
10434 && u8Val != 6 /* WB */
10435 && u8Val != 7 /* UC- */)
10436 {
10437 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10438 }
10439 u64Val >>= 8;
10440 }
10441 }
10442
10443 /*
10444 * EFER MSR.
10445 */
10446 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10447 {
10448 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10449 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10450 AssertRCBreak(rc);
10451 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10452 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10453 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10454 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10455 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10456 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10457 || !(u32GuestCr0 & X86_CR0_PG)
10458 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10459 VMX_IGS_EFER_LMA_LME_MISMATCH);
10460 }
10461
10462 /*
10463 * Segment registers.
10464 */
10465 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10466 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10467 if (!(u32Eflags & X86_EFL_VM))
10468 {
10469 /* CS */
10470 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10471 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10472 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10473 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10474 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10475 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10476 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10477 /* CS cannot be loaded with NULL in protected mode. */
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10479 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10480 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10481 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10482 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10483 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10484 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10485 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10486 else
10487 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10488
10489 /* SS */
10490 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10491 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10492 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10493 if ( !(pCtx->cr0 & X86_CR0_PE)
10494 || pCtx->cs.Attr.n.u4Type == 3)
10495 {
10496 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10497 }
10498 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10499 {
10500 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10501 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10502 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10503 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10504 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10505 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10506 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10507 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10508 }
10509
10510 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10511 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10512 {
10513 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10514 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10515 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10516 || pCtx->ds.Attr.n.u4Type > 11
10517 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10518 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10519 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10520 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10521 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10522 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10523 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10524 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10525 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10526 }
10527 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10528 {
10529 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10530 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10531 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10532 || pCtx->es.Attr.n.u4Type > 11
10533 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10534 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10535 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10536 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10537 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10538 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10539 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10540 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10541 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10542 }
10543 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10544 {
10545 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10546 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10547 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10548 || pCtx->fs.Attr.n.u4Type > 11
10549 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10550 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10551 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10552 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10553 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10554 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10555 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10556 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10557 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10558 }
10559 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10560 {
10561 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10562 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10563 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10564 || pCtx->gs.Attr.n.u4Type > 11
10565 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10566 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10567 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10568 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10569 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10570 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10571 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10572 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10573 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10574 }
10575 /* 64-bit capable CPUs. */
10576#if HC_ARCH_BITS == 64
10577 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10578 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10579 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10580 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10581 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10582 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10583 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10584 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10585 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10586 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10587 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10588#endif
10589 }
10590 else
10591 {
10592 /* V86 mode checks. */
10593 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10594 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10595 {
10596 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10597 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10598 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10599 }
10600 else
10601 {
10602 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10603 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10604 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10605 }
10606
10607 /* CS */
10608 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10609 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10610 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10611 /* SS */
10612 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10613 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10614 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10615 /* DS */
10616 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10617 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10618 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10619 /* ES */
10620 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10621 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10622 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10623 /* FS */
10624 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10625 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10626 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10627 /* GS */
10628 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10629 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10630 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10631 /* 64-bit capable CPUs. */
10632#if HC_ARCH_BITS == 64
10633 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10634 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10635 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10636 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10637 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10638 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10639 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10640 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10641 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10642 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10643 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10644#endif
10645 }
10646
10647 /*
10648 * TR.
10649 */
10650 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10651 /* 64-bit capable CPUs. */
10652#if HC_ARCH_BITS == 64
10653 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10654#endif
10655 if (fLongModeGuest)
10656 {
10657 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10658 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10659 }
10660 else
10661 {
10662 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10663 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10664 VMX_IGS_TR_ATTR_TYPE_INVALID);
10665 }
10666 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10667 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10668 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10669 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10670 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10671 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10672 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10673 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10674
10675 /*
10676 * GDTR and IDTR.
10677 */
10678#if HC_ARCH_BITS == 64
10679 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10680 AssertRCBreak(rc);
10681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10682
10683 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10684 AssertRCBreak(rc);
10685 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10686#endif
10687
10688 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10689 AssertRCBreak(rc);
10690 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10691
10692 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10693 AssertRCBreak(rc);
10694 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10695
10696 /*
10697 * Guest Non-Register State.
10698 */
10699 /* Activity State. */
10700 uint32_t u32ActivityState;
10701 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10702 AssertRCBreak(rc);
10703 HMVMX_CHECK_BREAK( !u32ActivityState
10704 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10705 VMX_IGS_ACTIVITY_STATE_INVALID);
10706 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10707 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10708 uint32_t u32IntrState;
10709 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10710 AssertRCBreak(rc);
10711 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10712 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10713 {
10714 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10715 }
10716
10717 /** @todo Activity state and injecting interrupts. Left as a todo since we
10718 * currently don't use activity states but ACTIVE. */
10719
10720 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10721 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10722
10723 /* Guest interruptibility-state. */
10724 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10725 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10726 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10727 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10728 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10729 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10730 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10731 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10732 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10733 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10734 {
10735 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10736 {
10737 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10738 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10739 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10740 }
10741 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10742 {
10743 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10744 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10745 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10746 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10747 }
10748 }
10749 /** @todo Assumes the processor is not in SMM. */
10750 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10751 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10752 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10753 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10754 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10755 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10756 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10757 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10758 {
10759 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10760 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10761 }
10762
10763 /* Pending debug exceptions. */
10764#if HC_ARCH_BITS == 64
10765 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10766 AssertRCBreak(rc);
10767 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10768 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10769 u32Val = u64Val; /* For pending debug exceptions checks below. */
10770#else
10771 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10772 AssertRCBreak(rc);
10773 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10774 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10775#endif
10776
10777 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10778 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10779 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10780 {
10781 if ( (u32Eflags & X86_EFL_TF)
10782 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10783 {
10784 /* Bit 14 is PendingDebug.BS. */
10785 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10786 }
10787 if ( !(u32Eflags & X86_EFL_TF)
10788 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10789 {
10790 /* Bit 14 is PendingDebug.BS. */
10791 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10792 }
10793 }
10794
10795 /* VMCS link pointer. */
10796 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10797 AssertRCBreak(rc);
10798 if (u64Val != UINT64_C(0xffffffffffffffff))
10799 {
10800 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10801 /** @todo Bits beyond the processor's physical-address width MBZ. */
10802 /** @todo 32-bit located in memory referenced by value of this field (as a
10803 * physical address) must contain the processor's VMCS revision ID. */
10804 /** @todo SMM checks. */
10805 }
10806
10807 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10808 * not using Nested Paging? */
10809 if ( pVM->hm.s.fNestedPaging
10810 && !fLongModeGuest
10811 && CPUMIsGuestInPAEModeEx(pCtx))
10812 {
10813 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10814 AssertRCBreak(rc);
10815 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10816
10817 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10818 AssertRCBreak(rc);
10819 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10820
10821 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10822 AssertRCBreak(rc);
10823 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10824
10825 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10826 AssertRCBreak(rc);
10827 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10828 }
10829
10830 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10831 if (uError == VMX_IGS_ERROR)
10832 uError = VMX_IGS_REASON_NOT_FOUND;
10833 } while (0);
10834
10835 pVCpu->hm.s.u32HMError = uError;
10836 return uError;
10837
10838#undef HMVMX_ERROR_BREAK
10839#undef HMVMX_CHECK_BREAK
10840}
10841
10842/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10843/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10844/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10845
10846/** @name VM-exit handlers.
10847 * @{
10848 */
10849
10850/**
10851 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10852 */
10853HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10854{
10855 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10857 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10858 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10859 return VINF_SUCCESS;
10860 return VINF_EM_RAW_INTERRUPT;
10861}
10862
10863
10864/**
10865 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10866 */
10867HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10868{
10869 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10870 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10871
10872 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10873 AssertRCReturn(rc, rc);
10874
10875 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10876 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10877 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10878 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10879
10880 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10881 {
10882 /*
10883 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10884 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10885 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10886 *
10887 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10888 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10889 */
10890 VMXDispatchHostNmi();
10891 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10892 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10893 return VINF_SUCCESS;
10894 }
10895
10896 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10897 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10898 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10899 { /* likely */ }
10900 else
10901 {
10902 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10903 rcStrictRc1 = VINF_SUCCESS;
10904 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10905 return rcStrictRc1;
10906 }
10907
10908 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10909 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10910 switch (uIntType)
10911 {
10912 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10913 Assert(uVector == X86_XCPT_DB);
10914 RT_FALL_THRU();
10915 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10916 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10917 RT_FALL_THRU();
10918 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10919 {
10920 /*
10921 * If there's any exception caused as a result of event injection, the resulting
10922 * secondary/final execption will be pending, we shall continue guest execution
10923 * after injecting the event. The page-fault case is complicated and we manually
10924 * handle any currently pending event in hmR0VmxExitXcptPF.
10925 */
10926 if (!pVCpu->hm.s.Event.fPending)
10927 { /* likely */ }
10928 else if (uVector != X86_XCPT_PF)
10929 {
10930 rc = VINF_SUCCESS;
10931 break;
10932 }
10933
10934 switch (uVector)
10935 {
10936 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10937 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10938 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10939 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10940 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10941 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10942
10943 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10944 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10945 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10946 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10947 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10948 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10949 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10950 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10951 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10952 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10953 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10954 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10955 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10956 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10957 default:
10958 {
10959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10960 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10961 {
10962 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10963 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10964 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10965
10966 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10967 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10968 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10969 AssertRCReturn(rc, rc);
10970 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10971 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10972 0 /* GCPtrFaultAddress */);
10973 }
10974 else
10975 {
10976 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10977 pVCpu->hm.s.u32HMError = uVector;
10978 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10979 }
10980 break;
10981 }
10982 }
10983 break;
10984 }
10985
10986 default:
10987 {
10988 pVCpu->hm.s.u32HMError = uExitIntInfo;
10989 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10990 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10991 break;
10992 }
10993 }
10994 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10995 return rc;
10996}
10997
10998
10999/**
11000 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11001 */
11002HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11003{
11004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11005
11006 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11007 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11008
11009 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11010 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11011 return VINF_SUCCESS;
11012}
11013
11014
11015/**
11016 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11017 */
11018HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11019{
11020 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11021 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11022 {
11023 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11024 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11025 }
11026
11027 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11028
11029 /*
11030 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11031 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11032 */
11033 uint32_t fIntrState = 0;
11034 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11035 AssertRCReturn(rc, rc);
11036
11037 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11038 if ( fBlockSti
11039 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11040 {
11041 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11042 }
11043
11044 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11045 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11046
11047 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11048 return VINF_SUCCESS;
11049}
11050
11051
11052/**
11053 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11054 */
11055HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11056{
11057 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11058 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11059}
11060
11061
11062/**
11063 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11064 */
11065HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11066{
11067 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11068 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11069}
11070
11071
11072/**
11073 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11074 */
11075HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11076{
11077 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11078
11079 /*
11080 * Get the state we need and update the exit history entry.
11081 */
11082 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11083 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11084 AssertRCReturn(rc, rc);
11085
11086 VBOXSTRICTRC rcStrict;
11087 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11088 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11089 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11090 if (!pExitRec)
11091 {
11092 /*
11093 * Regular CPUID instruction execution.
11094 */
11095 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11096 if (rcStrict == VINF_SUCCESS)
11097 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11098 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11099 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11100 {
11101 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11102 rcStrict = VINF_SUCCESS;
11103 }
11104 }
11105 else
11106 {
11107 /*
11108 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11109 */
11110 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11111 AssertRCReturn(rc2, rc2);
11112
11113 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11114 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11115
11116 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11117 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11118
11119 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11120 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11121 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11122 }
11123 return rcStrict;
11124}
11125
11126
11127/**
11128 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11129 */
11130HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11131{
11132 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11133 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11134 AssertRCReturn(rc, rc);
11135
11136 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11137 return VINF_EM_RAW_EMULATE_INSTR;
11138
11139 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11140 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11141}
11142
11143
11144/**
11145 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11146 */
11147HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11148{
11149 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11150 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11151 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11152 AssertRCReturn(rc, rc);
11153
11154 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11155 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11156 {
11157 /* If we get a spurious VM-exit when offsetting is enabled,
11158 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11159 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11160 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11161 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11162 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11163 }
11164 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11165 {
11166 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11167 rcStrict = VINF_SUCCESS;
11168 }
11169 return rcStrict;
11170}
11171
11172
11173/**
11174 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11175 */
11176HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11177{
11178 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11179 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11180 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11181 AssertRCReturn(rc, rc);
11182
11183 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11184 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11185 {
11186 /* If we get a spurious VM-exit when offsetting is enabled,
11187 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11188 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11189 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11190 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11191 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11192 }
11193 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11194 {
11195 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11196 rcStrict = VINF_SUCCESS;
11197 }
11198 return rcStrict;
11199}
11200
11201
11202/**
11203 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11204 */
11205HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11206{
11207 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11208 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11209 AssertRCReturn(rc, rc);
11210
11211 PVM pVM = pVCpu->CTX_SUFF(pVM);
11212 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11213 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11214 if (RT_LIKELY(rc == VINF_SUCCESS))
11215 {
11216 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11217 Assert(pVmxTransient->cbInstr == 2);
11218 }
11219 else
11220 {
11221 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11222 rc = VERR_EM_INTERPRETER;
11223 }
11224 return rc;
11225}
11226
11227
11228/**
11229 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11230 */
11231HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11232{
11233 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11234
11235 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11236 if (EMAreHypercallInstructionsEnabled(pVCpu))
11237 {
11238 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11239 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11240 AssertRCReturn(rc, rc);
11241
11242 /* Perform the hypercall. */
11243 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11244 if (rcStrict == VINF_SUCCESS)
11245 {
11246 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11247 AssertRCReturn(rc, rc);
11248 }
11249 else
11250 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11251 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11252 || RT_FAILURE(rcStrict));
11253
11254 /* If the hypercall changes anything other than guest's general-purpose registers,
11255 we would need to reload the guest changed bits here before VM-entry. */
11256 }
11257 else
11258 Log4Func(("Hypercalls not enabled\n"));
11259
11260 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11261 if (RT_FAILURE(rcStrict))
11262 {
11263 hmR0VmxSetPendingXcptUD(pVCpu);
11264 rcStrict = VINF_SUCCESS;
11265 }
11266
11267 return rcStrict;
11268}
11269
11270
11271/**
11272 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11273 */
11274HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11275{
11276 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11277 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11278
11279 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11280 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11281 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11282 AssertRCReturn(rc, rc);
11283
11284 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11285
11286 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11288 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11289 {
11290 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11291 rcStrict = VINF_SUCCESS;
11292 }
11293 else
11294 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11295 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11296 return rcStrict;
11297}
11298
11299
11300/**
11301 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11302 */
11303HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11304{
11305 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11306 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11307 AssertRCReturn(rc, rc);
11308
11309 PVM pVM = pVCpu->CTX_SUFF(pVM);
11310 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11311 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11312 if (RT_LIKELY(rc == VINF_SUCCESS))
11313 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11314 else
11315 {
11316 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11317 rc = VERR_EM_INTERPRETER;
11318 }
11319 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11320 return rc;
11321}
11322
11323
11324/**
11325 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11326 */
11327HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11328{
11329 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11330 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11331 AssertRCReturn(rc, rc);
11332
11333 PVM pVM = pVCpu->CTX_SUFF(pVM);
11334 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11335 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11336 rc = VBOXSTRICTRC_VAL(rc2);
11337 if (RT_LIKELY( rc == VINF_SUCCESS
11338 || rc == VINF_EM_HALT))
11339 {
11340 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11341 AssertRCReturn(rc3, rc3);
11342
11343 if ( rc == VINF_EM_HALT
11344 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11345 rc = VINF_SUCCESS;
11346 }
11347 else
11348 {
11349 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11350 rc = VERR_EM_INTERPRETER;
11351 }
11352 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11353 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11354 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11355 return rc;
11356}
11357
11358
11359/**
11360 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11361 */
11362HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11363{
11364 /*
11365 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11366 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11367 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11368 * VMX root operation. If we get here, something funny is going on.
11369 *
11370 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11371 */
11372 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11373 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11374 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11375}
11376
11377
11378/**
11379 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11380 */
11381HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11382{
11383 /*
11384 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11385 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11386 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11387 * an SMI. If we get here, something funny is going on.
11388 *
11389 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11390 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11391 */
11392 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11393 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11394 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11395}
11396
11397
11398/**
11399 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11400 */
11401HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11402{
11403 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11404 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11405 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11406 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11407}
11408
11409
11410/**
11411 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11412 */
11413HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11414{
11415 /*
11416 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11417 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11418 * See Intel spec. 25.3 "Other Causes of VM-exits".
11419 */
11420 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11421 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11422 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11423}
11424
11425
11426/**
11427 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11428 * VM-exit.
11429 */
11430HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11431{
11432 /*
11433 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11434 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11435 *
11436 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11437 * See Intel spec. "23.8 Restrictions on VMX operation".
11438 */
11439 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11440 return VINF_SUCCESS;
11441}
11442
11443
11444/**
11445 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11446 * VM-exit.
11447 */
11448HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11449{
11450 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11451 return VINF_EM_RESET;
11452}
11453
11454
11455/**
11456 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11457 */
11458HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11459{
11460 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11461 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11462
11463 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11464 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11465 AssertRCReturn(rc, rc);
11466
11467 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11468 rc = VINF_SUCCESS;
11469 else
11470 rc = VINF_EM_HALT;
11471
11472 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11473 if (rc != VINF_SUCCESS)
11474 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11475 return rc;
11476}
11477
11478
11479/**
11480 * VM-exit handler for instructions that result in a \#UD exception delivered to
11481 * the guest.
11482 */
11483HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11484{
11485 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11486 hmR0VmxSetPendingXcptUD(pVCpu);
11487 return VINF_SUCCESS;
11488}
11489
11490
11491/**
11492 * VM-exit handler for expiry of the VMX preemption timer.
11493 */
11494HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11495{
11496 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11497
11498 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11499 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11500
11501 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11502 PVM pVM = pVCpu->CTX_SUFF(pVM);
11503 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11505 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11506}
11507
11508
11509/**
11510 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11511 */
11512HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11513{
11514 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11515
11516 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11517 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11518 AssertRCReturn(rc, rc);
11519
11520 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11521 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11522 : HM_CHANGED_XCPT_RAISED_MASK);
11523
11524 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11525 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11526
11527 return rcStrict;
11528}
11529
11530
11531/**
11532 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11533 */
11534HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11535{
11536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11537 /** @todo Use VM-exit instruction information. */
11538 return VERR_EM_INTERPRETER;
11539}
11540
11541
11542/**
11543 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11544 * Error VM-exit.
11545 */
11546HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11547{
11548 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11549 AssertRCReturn(rc, rc);
11550 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11551 if (RT_FAILURE(rc))
11552 return rc;
11553
11554 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11555 NOREF(uInvalidReason);
11556
11557#ifdef VBOX_STRICT
11558 uint32_t fIntrState;
11559 RTHCUINTREG uHCReg;
11560 uint64_t u64Val;
11561 uint32_t u32Val;
11562
11563 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11564 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11565 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11566 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11567 AssertRCReturn(rc, rc);
11568
11569 Log4(("uInvalidReason %u\n", uInvalidReason));
11570 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11571 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11572 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11573 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11574
11575 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11576 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11577 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11578 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11579 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11580 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11581 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11582 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11583 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11584 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11585 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11586 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11587
11588 hmR0DumpRegs(pVCpu);
11589#else
11590 NOREF(pVmxTransient);
11591#endif
11592
11593 return VERR_VMX_INVALID_GUEST_STATE;
11594}
11595
11596
11597/**
11598 * VM-exit handler for VM-entry failure due to an MSR-load
11599 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11600 */
11601HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11602{
11603 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11604 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11605}
11606
11607
11608/**
11609 * VM-exit handler for VM-entry failure due to a machine-check event
11610 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11611 */
11612HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11613{
11614 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11615 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11616}
11617
11618
11619/**
11620 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11621 * theory.
11622 */
11623HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11624{
11625 RT_NOREF2(pVCpu, pVmxTransient);
11626 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11627 return VERR_VMX_UNDEFINED_EXIT_CODE;
11628}
11629
11630
11631/**
11632 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11633 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11634 * Conditional VM-exit.
11635 */
11636HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11637{
11638 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11639
11640 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11642 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11643 return VERR_EM_INTERPRETER;
11644 AssertMsgFailed(("Unexpected XDTR access\n"));
11645 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11646}
11647
11648
11649/**
11650 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11651 */
11652HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11653{
11654 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11655
11656 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11657 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11658 return VERR_EM_INTERPRETER;
11659 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11660 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11661}
11662
11663
11664/**
11665 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11666 */
11667HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11668{
11669 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11670
11671 /** @todo Optimize this: We currently drag in in the whole MSR state
11672 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11673 * MSRs required. That would require changes to IEM and possibly CPUM too.
11674 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11675 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11676 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11677 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11678 AssertRCReturn(rc, rc);
11679
11680 Log4Func(("ecx=%#RX32\n", idMsr));
11681
11682#ifdef VBOX_STRICT
11683 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11684 {
11685 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11686 && idMsr != MSR_K6_EFER)
11687 {
11688 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11689 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11690 }
11691 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11692 {
11693 VMXMSREXITREAD enmRead;
11694 VMXMSREXITWRITE enmWrite;
11695 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11696 AssertRCReturn(rc2, rc2);
11697 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11698 {
11699 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11700 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11701 }
11702 }
11703 }
11704#endif
11705
11706 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11707 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11708 if (rcStrict == VINF_SUCCESS)
11709 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11710 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11711 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11712 {
11713 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11714 rcStrict = VINF_SUCCESS;
11715 }
11716 else
11717 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11718
11719 return rcStrict;
11720}
11721
11722
11723/**
11724 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11725 */
11726HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11727{
11728 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11729
11730 /** @todo Optimize this: We currently drag in in the whole MSR state
11731 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11732 * MSRs required. That would require changes to IEM and possibly CPUM too.
11733 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11734 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11735 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11736 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11737 AssertRCReturn(rc, rc);
11738
11739 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11740
11741 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11742 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11743
11744 if (rcStrict == VINF_SUCCESS)
11745 {
11746 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11747
11748 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11749 if ( idMsr == MSR_IA32_APICBASE
11750 || ( idMsr >= MSR_IA32_X2APIC_START
11751 && idMsr <= MSR_IA32_X2APIC_END))
11752 {
11753 /*
11754 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11755 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11756 */
11757 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11758 }
11759 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11760 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11761 else if (idMsr == MSR_K6_EFER)
11762 {
11763 /*
11764 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11765 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11766 * the other bits as well, SCE and NXE. See @bugref{7368}.
11767 */
11768 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11769 | HM_CHANGED_VMX_EXIT_CTLS);
11770 }
11771
11772 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11773 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11774 {
11775 switch (idMsr)
11776 {
11777 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11778 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11779 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11780 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11781 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11782 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11783 default:
11784 {
11785 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11786 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11787 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11788 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11789 break;
11790 }
11791 }
11792 }
11793#ifdef VBOX_STRICT
11794 else
11795 {
11796 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11797 switch (idMsr)
11798 {
11799 case MSR_IA32_SYSENTER_CS:
11800 case MSR_IA32_SYSENTER_EIP:
11801 case MSR_IA32_SYSENTER_ESP:
11802 case MSR_K8_FS_BASE:
11803 case MSR_K8_GS_BASE:
11804 {
11805 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11806 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11807 }
11808
11809 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11810 default:
11811 {
11812 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11813 {
11814 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11815 if (idMsr != MSR_K6_EFER)
11816 {
11817 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11818 idMsr));
11819 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11820 }
11821 }
11822
11823 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11824 {
11825 VMXMSREXITREAD enmRead;
11826 VMXMSREXITWRITE enmWrite;
11827 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11828 AssertRCReturn(rc2, rc2);
11829 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11830 {
11831 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11832 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11833 }
11834 }
11835 break;
11836 }
11837 }
11838 }
11839#endif /* VBOX_STRICT */
11840 }
11841 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11842 {
11843 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11844 rcStrict = VINF_SUCCESS;
11845 }
11846 else
11847 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11848
11849 return rcStrict;
11850}
11851
11852
11853/**
11854 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11855 */
11856HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11857{
11858 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11859 /** @todo The guest has likely hit a contended spinlock. We might want to
11860 * poke a schedule different guest VCPU. */
11861 return VINF_EM_RAW_INTERRUPT;
11862}
11863
11864
11865/**
11866 * VM-exit handler for when the TPR value is lowered below the specified
11867 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11868 */
11869HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11870{
11871 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11872 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11873
11874 /*
11875 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11876 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11877 */
11878 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11879 return VINF_SUCCESS;
11880}
11881
11882
11883/**
11884 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11885 * VM-exit.
11886 *
11887 * @retval VINF_SUCCESS when guest execution can continue.
11888 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11889 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11890 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11891 * interpreter.
11892 */
11893HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11894{
11895 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11896 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11897
11898 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11899 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11900 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11901 AssertRCReturn(rc, rc);
11902
11903 VBOXSTRICTRC rcStrict;
11904 PVM pVM = pVCpu->CTX_SUFF(pVM);
11905 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11906 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11907 switch (uAccessType)
11908 {
11909 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11910 {
11911 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11912 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11913 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11914 AssertMsg( rcStrict == VINF_SUCCESS
11915 || rcStrict == VINF_IEM_RAISED_XCPT
11916 || rcStrict == VINF_PGM_CHANGE_MODE
11917 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11918
11919 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11920 {
11921 case 0:
11922 {
11923 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11924 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11925 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11926 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11927 break;
11928 }
11929
11930 case 2:
11931 {
11932 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11933 /* Nothing to do here, CR2 it's not part of the VMCS. */
11934 break;
11935 }
11936
11937 case 3:
11938 {
11939 Assert( !pVM->hm.s.fNestedPaging
11940 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11941 || pVCpu->hm.s.fUsingDebugLoop);
11942 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11943 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11944 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11945 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11946 break;
11947 }
11948
11949 case 4:
11950 {
11951 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11952 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11953 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11954 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11955 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11956 break;
11957 }
11958
11959 case 8:
11960 {
11961 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11962 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11963 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11964 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11965 break;
11966 }
11967 default:
11968 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11969 break;
11970 }
11971 break;
11972 }
11973
11974 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11975 {
11976 Assert( !pVM->hm.s.fNestedPaging
11977 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11978 || pVCpu->hm.s.fUsingDebugLoop
11979 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11980 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11981 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11982 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11983
11984 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11985 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11986 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11987 AssertMsg( rcStrict == VINF_SUCCESS
11988 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11989#ifdef VBOX_WITH_STATISTICS
11990 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11991 {
11992 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11993 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11994 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11995 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11996 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11997 }
11998#endif
11999 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12000 VBOXSTRICTRC_VAL(rcStrict)));
12001 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12002 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12003 else
12004 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12005 break;
12006 }
12007
12008 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12009 {
12010 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12011 AssertMsg( rcStrict == VINF_SUCCESS
12012 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12013
12014 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12015 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12016 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12017 break;
12018 }
12019
12020 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12021 {
12022 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12023 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12024 AssertMsg( rcStrict == VINF_SUCCESS
12025 || rcStrict == VINF_IEM_RAISED_XCPT
12026 || rcStrict == VINF_PGM_CHANGE_MODE,
12027 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12028
12029 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12031 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12032 break;
12033 }
12034
12035 default:
12036 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12037 VERR_VMX_UNEXPECTED_EXCEPTION);
12038 }
12039
12040 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12041 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12042 if (rcStrict == VINF_IEM_RAISED_XCPT)
12043 {
12044 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
12045 rcStrict = VINF_SUCCESS;
12046 }
12047
12048 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12049 NOREF(pVM);
12050 return rcStrict;
12051}
12052
12053
12054/**
12055 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12056 * VM-exit.
12057 */
12058HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12059{
12060 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12061 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12062
12063 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12064 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12065 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12066 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12067 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12068 AssertRCReturn(rc, rc);
12069
12070 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12071 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12072 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12073 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12074 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12075 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12076 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12077 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12078 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12079
12080 /*
12081 * Update exit history to see if this exit can be optimized.
12082 */
12083 VBOXSTRICTRC rcStrict;
12084 PCEMEXITREC pExitRec = NULL;
12085 if ( !fGstStepping
12086 && !fDbgStepping)
12087 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12088 !fIOString
12089 ? !fIOWrite
12090 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12091 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12092 : !fIOWrite
12093 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12094 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12095 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12096 if (!pExitRec)
12097 {
12098 /* I/O operation lookup arrays. */
12099 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12100 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12101 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12102 uint32_t const cbInstr = pVmxTransient->cbInstr;
12103 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12104 PVM pVM = pVCpu->CTX_SUFF(pVM);
12105 if (fIOString)
12106 {
12107 /*
12108 * INS/OUTS - I/O String instruction.
12109 *
12110 * Use instruction-information if available, otherwise fall back on
12111 * interpreting the instruction.
12112 */
12113 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12114 fIOWrite ? 'w' : 'r'));
12115 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12116 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12117 {
12118 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12119 AssertRCReturn(rc2, rc2);
12120 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12121 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12122 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12123 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12124 if (fIOWrite)
12125 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12126 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12127 else
12128 {
12129 /*
12130 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12131 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12132 * See Intel Instruction spec. for "INS".
12133 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12134 */
12135 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12136 }
12137 }
12138 else
12139 rcStrict = IEMExecOne(pVCpu);
12140
12141 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12142 fUpdateRipAlready = true;
12143 }
12144 else
12145 {
12146 /*
12147 * IN/OUT - I/O instruction.
12148 */
12149 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12150 fIOWrite ? 'w' : 'r'));
12151 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12152 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12153 if (fIOWrite)
12154 {
12155 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12156 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12157 }
12158 else
12159 {
12160 uint32_t u32Result = 0;
12161 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12162 if (IOM_SUCCESS(rcStrict))
12163 {
12164 /* Save result of I/O IN instr. in AL/AX/EAX. */
12165 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12166 }
12167 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12168 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12169 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12170 }
12171 }
12172
12173 if (IOM_SUCCESS(rcStrict))
12174 {
12175 if (!fUpdateRipAlready)
12176 {
12177 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12178 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12179 }
12180
12181 /*
12182 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12183 * while booting Fedora 17 64-bit guest.
12184 *
12185 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12186 */
12187 if (fIOString)
12188 {
12189 /** @todo Single-step for INS/OUTS with REP prefix? */
12190 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12191 }
12192 else if ( !fDbgStepping
12193 && fGstStepping)
12194 {
12195 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12196 AssertRCReturn(rc, rc);
12197 }
12198
12199 /*
12200 * If any I/O breakpoints are armed, we need to check if one triggered
12201 * and take appropriate action.
12202 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12203 */
12204 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12205 AssertRCReturn(rc, rc);
12206
12207 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12208 * execution engines about whether hyper BPs and such are pending. */
12209 uint32_t const uDr7 = pCtx->dr[7];
12210 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12211 && X86_DR7_ANY_RW_IO(uDr7)
12212 && (pCtx->cr4 & X86_CR4_DE))
12213 || DBGFBpIsHwIoArmed(pVM)))
12214 {
12215 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12216
12217 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12218 VMMRZCallRing3Disable(pVCpu);
12219 HM_DISABLE_PREEMPT(pVCpu);
12220
12221 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12222
12223 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12224 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12225 {
12226 /* Raise #DB. */
12227 if (fIsGuestDbgActive)
12228 ASMSetDR6(pCtx->dr[6]);
12229 if (pCtx->dr[7] != uDr7)
12230 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12231
12232 hmR0VmxSetPendingXcptDB(pVCpu);
12233 }
12234 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12235 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12236 else if ( rcStrict2 != VINF_SUCCESS
12237 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12238 rcStrict = rcStrict2;
12239 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12240
12241 HM_RESTORE_PREEMPT();
12242 VMMRZCallRing3Enable(pVCpu);
12243 }
12244 }
12245
12246#ifdef VBOX_STRICT
12247 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12248 Assert(!fIOWrite);
12249 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12250 Assert(fIOWrite);
12251 else
12252 {
12253# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12254 * statuses, that the VMM device and some others may return. See
12255 * IOM_SUCCESS() for guidance. */
12256 AssertMsg( RT_FAILURE(rcStrict)
12257 || rcStrict == VINF_SUCCESS
12258 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12259 || rcStrict == VINF_EM_DBG_BREAKPOINT
12260 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12261 || rcStrict == VINF_EM_RAW_TO_R3
12262 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12263# endif
12264 }
12265#endif
12266 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12267 }
12268 else
12269 {
12270 /*
12271 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12272 */
12273 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12274 AssertRCReturn(rc2, rc2);
12275 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12276 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12277 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12278 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12279 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12280 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12281
12282 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12283 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12284
12285 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12286 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12287 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12288 }
12289 return rcStrict;
12290}
12291
12292
12293/**
12294 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12295 * VM-exit.
12296 */
12297HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12298{
12299 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12300
12301 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12302 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12303 AssertRCReturn(rc, rc);
12304 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12305 {
12306 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12307 AssertRCReturn(rc, rc);
12308 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12309 {
12310 uint32_t uErrCode;
12311 RTGCUINTPTR GCPtrFaultAddress;
12312 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12313 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12314 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12315 if (fErrorCodeValid)
12316 {
12317 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12318 AssertRCReturn(rc, rc);
12319 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12320 }
12321 else
12322 uErrCode = 0;
12323
12324 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12325 && uVector == X86_XCPT_PF)
12326 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12327 else
12328 GCPtrFaultAddress = 0;
12329
12330 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12331 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12332
12333 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12334 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12335 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12336 }
12337 }
12338
12339 /* Fall back to the interpreter to emulate the task-switch. */
12340 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12341 return VERR_EM_INTERPRETER;
12342}
12343
12344
12345/**
12346 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12347 */
12348HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12349{
12350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12351 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12352 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12353 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12354 AssertRCReturn(rc, rc);
12355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12356 return VINF_EM_DBG_STEPPED;
12357}
12358
12359
12360/**
12361 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12362 */
12363HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12364{
12365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12366
12367 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12368
12369 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12370 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12371 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12372 {
12373 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12374 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12375 {
12376 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12377 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12378 }
12379 }
12380 else
12381 {
12382 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12383 rcStrict1 = VINF_SUCCESS;
12384 return rcStrict1;
12385 }
12386
12387 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12388 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12389 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12390 AssertRCReturn(rc, rc);
12391
12392 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12393 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12394 VBOXSTRICTRC rcStrict2;
12395 switch (uAccessType)
12396 {
12397 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12398 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12399 {
12400 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12401 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12402 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12403
12404 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12405 GCPhys &= PAGE_BASE_GC_MASK;
12406 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12407 PVM pVM = pVCpu->CTX_SUFF(pVM);
12408 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12409 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12410
12411 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12412 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12413 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12414 CPUMCTX2CORE(pCtx), GCPhys);
12415 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12416 if ( rcStrict2 == VINF_SUCCESS
12417 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12418 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12419 {
12420 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12421 | HM_CHANGED_GUEST_APIC_TPR);
12422 rcStrict2 = VINF_SUCCESS;
12423 }
12424 break;
12425 }
12426
12427 default:
12428 Log4Func(("uAccessType=%#x\n", uAccessType));
12429 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12430 break;
12431 }
12432
12433 if (rcStrict2 != VINF_SUCCESS)
12434 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12435 return rcStrict2;
12436}
12437
12438
12439/**
12440 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12441 * VM-exit.
12442 */
12443HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12444{
12445 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12446
12447 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12448 if (pVmxTransient->fWasGuestDebugStateActive)
12449 {
12450 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12451 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12452 }
12453
12454 if ( !pVCpu->hm.s.fSingleInstruction
12455 && !pVmxTransient->fWasHyperDebugStateActive)
12456 {
12457 Assert(!DBGFIsStepping(pVCpu));
12458 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12459
12460 /* Don't intercept MOV DRx any more. */
12461 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12462 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12463 AssertRCReturn(rc, rc);
12464
12465 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12466 VMMRZCallRing3Disable(pVCpu);
12467 HM_DISABLE_PREEMPT(pVCpu);
12468
12469 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12470 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12471 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12472
12473 HM_RESTORE_PREEMPT();
12474 VMMRZCallRing3Enable(pVCpu);
12475
12476#ifdef VBOX_WITH_STATISTICS
12477 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12478 AssertRCReturn(rc, rc);
12479 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12481 else
12482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12483#endif
12484 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12485 return VINF_SUCCESS;
12486 }
12487
12488 /*
12489 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12490 * Update the segment registers and DR7 from the CPU.
12491 */
12492 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12493 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12494 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12495 AssertRCReturn(rc, rc);
12496 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12497
12498 PVM pVM = pVCpu->CTX_SUFF(pVM);
12499 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12500 {
12501 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12502 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12503 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12504 if (RT_SUCCESS(rc))
12505 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12506 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12507 }
12508 else
12509 {
12510 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12511 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12512 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12513 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12514 }
12515
12516 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12517 if (RT_SUCCESS(rc))
12518 {
12519 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12520 AssertRCReturn(rc2, rc2);
12521 return VINF_SUCCESS;
12522 }
12523 return rc;
12524}
12525
12526
12527/**
12528 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12529 * Conditional VM-exit.
12530 */
12531HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12532{
12533 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12534 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12535
12536 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12537 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12538 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12539 {
12540 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12541 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12542 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12543 {
12544 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12545 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12546 }
12547 }
12548 else
12549 {
12550 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12551 rcStrict1 = VINF_SUCCESS;
12552 return rcStrict1;
12553 }
12554
12555 /*
12556 * Get sufficent state and update the exit history entry.
12557 */
12558 RTGCPHYS GCPhys;
12559 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12560 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12561 AssertRCReturn(rc, rc);
12562
12563 VBOXSTRICTRC rcStrict;
12564 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12565 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12566 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12567 if (!pExitRec)
12568 {
12569 /*
12570 * If we succeed, resume guest execution.
12571 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12572 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12573 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12574 * weird case. See @bugref{6043}.
12575 */
12576 PVM pVM = pVCpu->CTX_SUFF(pVM);
12577 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12578 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12579 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12580 if ( rcStrict == VINF_SUCCESS
12581 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12582 || rcStrict == VERR_PAGE_NOT_PRESENT)
12583 {
12584 /* Successfully handled MMIO operation. */
12585 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12586 | HM_CHANGED_GUEST_APIC_TPR);
12587 rcStrict = VINF_SUCCESS;
12588 }
12589 }
12590 else
12591 {
12592 /*
12593 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12594 */
12595 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12596 AssertRCReturn(rc2, rc2);
12597
12598 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12599 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12600
12601 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12602 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12603
12604 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12605 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12606 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12607 }
12608 return VBOXSTRICTRC_TODO(rcStrict);
12609}
12610
12611
12612/**
12613 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12614 * VM-exit.
12615 */
12616HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12617{
12618 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12619 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12620
12621 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12622 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12623 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12624 {
12625 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12626 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12627 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12628 }
12629 else
12630 {
12631 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12632 rcStrict1 = VINF_SUCCESS;
12633 return rcStrict1;
12634 }
12635
12636 RTGCPHYS GCPhys;
12637 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12638 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12639 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12640 AssertRCReturn(rc, rc);
12641
12642 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12643 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12644
12645 RTGCUINT uErrorCode = 0;
12646 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12647 uErrorCode |= X86_TRAP_PF_ID;
12648 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12649 uErrorCode |= X86_TRAP_PF_RW;
12650 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12651 uErrorCode |= X86_TRAP_PF_P;
12652
12653 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12654
12655
12656 /* Handle the pagefault trap for the nested shadow table. */
12657 PVM pVM = pVCpu->CTX_SUFF(pVM);
12658 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12659
12660 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12661 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12662
12663 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12664 TRPMResetTrap(pVCpu);
12665
12666 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12667 if ( rcStrict2 == VINF_SUCCESS
12668 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12669 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12670 {
12671 /* Successfully synced our nested page tables. */
12672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12673 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12674 return VINF_SUCCESS;
12675 }
12676
12677 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12678 return rcStrict2;
12679}
12680
12681/** @} */
12682
12683/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12684/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12685/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12686
12687/** @name VM-exit exception handlers.
12688 * @{
12689 */
12690
12691/**
12692 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12693 */
12694static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12695{
12696 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12698
12699 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12700 AssertRCReturn(rc, rc);
12701
12702 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12703 {
12704 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12705 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12706
12707 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12708 * provides VM-exit instruction length. If this causes problem later,
12709 * disassemble the instruction like it's done on AMD-V. */
12710 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12711 AssertRCReturn(rc2, rc2);
12712 return rc;
12713 }
12714
12715 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12716 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12717 return rc;
12718}
12719
12720
12721/**
12722 * VM-exit exception handler for \#BP (Breakpoint exception).
12723 */
12724static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12725{
12726 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12727 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12728
12729 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12730 AssertRCReturn(rc, rc);
12731
12732 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12733 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12734 if (rc == VINF_EM_RAW_GUEST_TRAP)
12735 {
12736 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12737 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12738 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12739 AssertRCReturn(rc, rc);
12740
12741 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12742 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12743 }
12744
12745 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12746 return rc;
12747}
12748
12749
12750/**
12751 * VM-exit exception handler for \#AC (alignment check exception).
12752 */
12753static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12754{
12755 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12756
12757 /*
12758 * Re-inject it. We'll detect any nesting before getting here.
12759 */
12760 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12761 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12762 AssertRCReturn(rc, rc);
12763 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12764
12765 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12766 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12767 return VINF_SUCCESS;
12768}
12769
12770
12771/**
12772 * VM-exit exception handler for \#DB (Debug exception).
12773 */
12774static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12775{
12776 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12778
12779 /*
12780 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12781 * for processing.
12782 */
12783 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12784
12785 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12786 uint64_t uDR6 = X86_DR6_INIT_VAL;
12787 uDR6 |= ( pVmxTransient->uExitQualification
12788 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12789
12790 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12791 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12792 Log6Func(("rc=%Rrc\n", rc));
12793 if (rc == VINF_EM_RAW_GUEST_TRAP)
12794 {
12795 /*
12796 * The exception was for the guest. Update DR6, DR7.GD and
12797 * IA32_DEBUGCTL.LBR before forwarding it.
12798 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12799 */
12800 VMMRZCallRing3Disable(pVCpu);
12801 HM_DISABLE_PREEMPT(pVCpu);
12802
12803 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12804 pCtx->dr[6] |= uDR6;
12805 if (CPUMIsGuestDebugStateActive(pVCpu))
12806 ASMSetDR6(pCtx->dr[6]);
12807
12808 HM_RESTORE_PREEMPT();
12809 VMMRZCallRing3Enable(pVCpu);
12810
12811 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12812 AssertRCReturn(rc, rc);
12813
12814 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12815 pCtx->dr[7] &= ~X86_DR7_GD;
12816
12817 /* Paranoia. */
12818 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12819 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12820
12821 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12822 AssertRCReturn(rc, rc);
12823
12824 /*
12825 * Raise #DB in the guest.
12826 *
12827 * It is important to reflect exactly what the VM-exit gave us (preserving the
12828 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12829 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12830 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12831 *
12832 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12833 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12834 */
12835 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12836 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12837 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12838 AssertRCReturn(rc, rc);
12839 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12840 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12841 return VINF_SUCCESS;
12842 }
12843
12844 /*
12845 * Not a guest trap, must be a hypervisor related debug event then.
12846 * Update DR6 in case someone is interested in it.
12847 */
12848 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12849 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12850 CPUMSetHyperDR6(pVCpu, uDR6);
12851
12852 return rc;
12853}
12854
12855/**
12856 * VM-exit exception handler for \#GP (General-protection exception).
12857 *
12858 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12859 */
12860static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12861{
12862 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12863 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12864
12865 int rc;
12866 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12867 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12868 { /* likely */ }
12869 else
12870 {
12871#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12872 Assert(pVCpu->hm.s.fUsingDebugLoop);
12873#endif
12874 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12875 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12876 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12877 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12878 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12879 AssertRCReturn(rc, rc);
12880 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12881 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12882 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12883 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12884 return rc;
12885 }
12886
12887 Assert(CPUMIsGuestInRealModeEx(pCtx));
12888 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12889
12890 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12891 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12892 AssertRCReturn(rc, rc);
12893
12894 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12895 uint32_t cbOp = 0;
12896 PVM pVM = pVCpu->CTX_SUFF(pVM);
12897 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12898 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12899 if (RT_SUCCESS(rc))
12900 {
12901 rc = VINF_SUCCESS;
12902 Assert(cbOp == pDis->cbInstr);
12903 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12904 switch (pDis->pCurInstr->uOpcode)
12905 {
12906 case OP_CLI:
12907 {
12908 pCtx->eflags.Bits.u1IF = 0;
12909 pCtx->eflags.Bits.u1RF = 0;
12910 pCtx->rip += pDis->cbInstr;
12911 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12912 if ( !fDbgStepping
12913 && pCtx->eflags.Bits.u1TF)
12914 {
12915 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12916 AssertRCReturn(rc, rc);
12917 }
12918 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12919 break;
12920 }
12921
12922 case OP_STI:
12923 {
12924 bool fOldIF = pCtx->eflags.Bits.u1IF;
12925 pCtx->eflags.Bits.u1IF = 1;
12926 pCtx->eflags.Bits.u1RF = 0;
12927 pCtx->rip += pDis->cbInstr;
12928 if (!fOldIF)
12929 {
12930 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12931 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12932 }
12933 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12934 if ( !fDbgStepping
12935 && pCtx->eflags.Bits.u1TF)
12936 {
12937 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12938 AssertRCReturn(rc, rc);
12939 }
12940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12941 break;
12942 }
12943
12944 case OP_HLT:
12945 {
12946 rc = VINF_EM_HALT;
12947 pCtx->rip += pDis->cbInstr;
12948 pCtx->eflags.Bits.u1RF = 0;
12949 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12950 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12951 break;
12952 }
12953
12954 case OP_POPF:
12955 {
12956 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12957 uint32_t cbParm;
12958 uint32_t uMask;
12959 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12960 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12961 {
12962 cbParm = 4;
12963 uMask = 0xffffffff;
12964 }
12965 else
12966 {
12967 cbParm = 2;
12968 uMask = 0xffff;
12969 }
12970
12971 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12972 RTGCPTR GCPtrStack = 0;
12973 X86EFLAGS Eflags;
12974 Eflags.u32 = 0;
12975 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12976 &GCPtrStack);
12977 if (RT_SUCCESS(rc))
12978 {
12979 Assert(sizeof(Eflags.u32) >= cbParm);
12980 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12981 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12982 }
12983 if (RT_FAILURE(rc))
12984 {
12985 rc = VERR_EM_INTERPRETER;
12986 break;
12987 }
12988 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
12989 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
12990 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12991 pCtx->esp += cbParm;
12992 pCtx->esp &= uMask;
12993 pCtx->rip += pDis->cbInstr;
12994 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12995 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12996 POPF restores EFLAGS.TF. */
12997 if ( !fDbgStepping
12998 && fGstStepping)
12999 {
13000 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13001 AssertRCReturn(rc, rc);
13002 }
13003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13004 break;
13005 }
13006
13007 case OP_PUSHF:
13008 {
13009 uint32_t cbParm;
13010 uint32_t uMask;
13011 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13012 {
13013 cbParm = 4;
13014 uMask = 0xffffffff;
13015 }
13016 else
13017 {
13018 cbParm = 2;
13019 uMask = 0xffff;
13020 }
13021
13022 /* Get the stack pointer & push the contents of eflags onto the stack. */
13023 RTGCPTR GCPtrStack = 0;
13024 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13025 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13026 if (RT_FAILURE(rc))
13027 {
13028 rc = VERR_EM_INTERPRETER;
13029 break;
13030 }
13031 X86EFLAGS Eflags = pCtx->eflags;
13032 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13033 Eflags.Bits.u1RF = 0;
13034 Eflags.Bits.u1VM = 0;
13035
13036 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13037 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13038 {
13039 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13040 rc = VERR_EM_INTERPRETER;
13041 break;
13042 }
13043 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13044 pCtx->esp -= cbParm;
13045 pCtx->esp &= uMask;
13046 pCtx->rip += pDis->cbInstr;
13047 pCtx->eflags.Bits.u1RF = 0;
13048 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13049 if ( !fDbgStepping
13050 && pCtx->eflags.Bits.u1TF)
13051 {
13052 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13053 AssertRCReturn(rc, rc);
13054 }
13055 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13056 break;
13057 }
13058
13059 case OP_IRET:
13060 {
13061 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13062 * instruction reference. */
13063 RTGCPTR GCPtrStack = 0;
13064 uint32_t uMask = 0xffff;
13065 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13066 uint16_t aIretFrame[3];
13067 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13068 {
13069 rc = VERR_EM_INTERPRETER;
13070 break;
13071 }
13072 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13073 &GCPtrStack);
13074 if (RT_SUCCESS(rc))
13075 {
13076 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13077 PGMACCESSORIGIN_HM));
13078 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13079 }
13080 if (RT_FAILURE(rc))
13081 {
13082 rc = VERR_EM_INTERPRETER;
13083 break;
13084 }
13085 pCtx->eip = 0;
13086 pCtx->ip = aIretFrame[0];
13087 pCtx->cs.Sel = aIretFrame[1];
13088 pCtx->cs.ValidSel = aIretFrame[1];
13089 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13090 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13091 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13092 pCtx->sp += sizeof(aIretFrame);
13093 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13094 | HM_CHANGED_GUEST_CS);
13095 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13096 if ( !fDbgStepping
13097 && fGstStepping)
13098 {
13099 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13100 AssertRCReturn(rc, rc);
13101 }
13102 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13103 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13104 break;
13105 }
13106
13107 case OP_INT:
13108 {
13109 uint16_t uVector = pDis->Param1.uValue & 0xff;
13110 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13111 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13112 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13113 break;
13114 }
13115
13116 case OP_INTO:
13117 {
13118 if (pCtx->eflags.Bits.u1OF)
13119 {
13120 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13121 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13123 }
13124 else
13125 {
13126 pCtx->eflags.Bits.u1RF = 0;
13127 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13128 }
13129 break;
13130 }
13131
13132 default:
13133 {
13134 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13135 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13136 EMCODETYPE_SUPERVISOR);
13137 rc = VBOXSTRICTRC_VAL(rc2);
13138 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13139 /** @todo We have to set pending-debug exceptions here when the guest is
13140 * single-stepping depending on the instruction that was interpreted. */
13141 Log4Func(("#GP rc=%Rrc\n", rc));
13142 break;
13143 }
13144 }
13145 }
13146 else
13147 rc = VERR_EM_INTERPRETER;
13148
13149 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13150 ("#GP Unexpected rc=%Rrc\n", rc));
13151 return rc;
13152}
13153
13154
13155/**
13156 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13157 * the exception reported in the VMX transient structure back into the VM.
13158 *
13159 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13160 * up-to-date.
13161 */
13162static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13163{
13164 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13165#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13166 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13167 ("uVector=%#x u32XcptBitmap=%#X32\n",
13168 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13169#endif
13170
13171 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13172 hmR0VmxCheckExitDueToEventDelivery(). */
13173 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13174 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13175 AssertRCReturn(rc, rc);
13176 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13177
13178#ifdef DEBUG_ramshankar
13179 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13180 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13181 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13182#endif
13183
13184 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13185 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13186 return VINF_SUCCESS;
13187}
13188
13189
13190/**
13191 * VM-exit exception handler for \#PF (Page-fault exception).
13192 */
13193static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13194{
13195 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13196 PVM pVM = pVCpu->CTX_SUFF(pVM);
13197 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13198 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13199 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13200 AssertRCReturn(rc, rc);
13201
13202 if (!pVM->hm.s.fNestedPaging)
13203 { /* likely */ }
13204 else
13205 {
13206#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13207 Assert(pVCpu->hm.s.fUsingDebugLoop);
13208#endif
13209 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13210 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13211 {
13212 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13213 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13214 }
13215 else
13216 {
13217 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13218 hmR0VmxSetPendingXcptDF(pVCpu);
13219 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13220 }
13221 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13222 return rc;
13223 }
13224
13225 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13226 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13227 if (pVmxTransient->fVectoringPF)
13228 {
13229 Assert(pVCpu->hm.s.Event.fPending);
13230 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13231 }
13232
13233 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13234 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13235 AssertRCReturn(rc, rc);
13236
13237 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13238 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13239
13240 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13241 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13242 (RTGCPTR)pVmxTransient->uExitQualification);
13243
13244 Log4Func(("#PF: rc=%Rrc\n", rc));
13245 if (rc == VINF_SUCCESS)
13246 {
13247 /*
13248 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13249 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13250 */
13251 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13252 TRPMResetTrap(pVCpu);
13253 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13254 return rc;
13255 }
13256
13257 if (rc == VINF_EM_RAW_GUEST_TRAP)
13258 {
13259 if (!pVmxTransient->fVectoringDoublePF)
13260 {
13261 /* It's a guest page fault and needs to be reflected to the guest. */
13262 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13263 TRPMResetTrap(pVCpu);
13264 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13265 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13266 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13267 }
13268 else
13269 {
13270 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13271 TRPMResetTrap(pVCpu);
13272 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13273 hmR0VmxSetPendingXcptDF(pVCpu);
13274 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13275 }
13276
13277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13278 return VINF_SUCCESS;
13279 }
13280
13281 TRPMResetTrap(pVCpu);
13282 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13283 return rc;
13284}
13285
13286/** @} */
13287
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette