VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72988

Last change on this file since 72988 was 72988, checked in by vboxsync, 6 years ago

VMM/HMVMXR0, HMSVMR0: Eliminate hidden parameters from a few macros and other minor nits.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 563.3 KB
Line 
1/* $Id: HMVMXR0.cpp 72988 2018-07-09 02:27:49Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Loads a set of guests MSRs to allow read/passthru to the guest.
1524 *
1525 * The name of this function is slightly confusing. This function does NOT
1526 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527 * common prefix for functions dealing with "lazy restoration" of the shared
1528 * MSRs.
1529 *
1530 * @param pVCpu The cross context virtual CPU structure.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 /*
1544 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546 * we can skip a few MSR writes.
1547 *
1548 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549 * guest MSR values in the guest-CPU context might be different to what's currently
1550 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551 * CPU, see @bugref{8728}.
1552 */
1553 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559 {
1560#ifdef VBOX_STRICT
1561 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565#endif
1566 }
1567 else
1568 {
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573 }
1574 }
1575#endif
1576 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxImportGuestState()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598#if HC_ARCH_BITS == 64
1599 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600 {
1601 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605 }
1606#endif
1607 }
1608 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1609}
1610
1611
1612/**
1613 * Verifies that our cached values of the VMCS fields are all consistent with
1614 * what's actually present in the VMCS.
1615 *
1616 * @returns VBox status code.
1617 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619 * VMCS content. HMCPU error-field is
1620 * updated, see VMX_VCI_XXX.
1621 * @param pVCpu The cross context virtual CPU structure.
1622 */
1623static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624{
1625 uint32_t u32Val;
1626 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627 AssertRCReturn(rc, rc);
1628 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655 {
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662 }
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671 uint64_t u64Val;
1672 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673 AssertRCReturn(rc, rc);
1674 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679 return VINF_SUCCESS;
1680}
1681
1682
1683#ifdef VBOX_STRICT
1684/**
1685 * Verifies that our cached host EFER value has not changed
1686 * since we cached it.
1687 *
1688 * @param pVCpu The cross context virtual CPU structure.
1689 */
1690static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691{
1692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695 {
1696 uint64_t u64Val;
1697 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698 AssertRC(rc);
1699
1700 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702 }
1703}
1704
1705
1706/**
1707 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708 * VMCS are correct.
1709 *
1710 * @param pVCpu The cross context virtual CPU structure.
1711 */
1712static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713{
1714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716 /* Verify MSR counts in the VMCS are what we think it should be. */
1717 uint32_t cMsrs;
1718 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730 {
1731 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733 pGuestMsr->u32Msr, cMsrs));
1734
1735 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739 /* Verify that the permissions are as expected in the MSR bitmap. */
1740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741 {
1742 VMXMSREXITREAD enmRead;
1743 VMXMSREXITWRITE enmWrite;
1744 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747 {
1748 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750 }
1751 else
1752 {
1753 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756 pGuestMsr->u32Msr, cMsrs));
1757 }
1758 }
1759 }
1760}
1761#endif /* VBOX_STRICT */
1762
1763
1764/**
1765 * Flushes the TLB using EPT.
1766 *
1767 * @returns VBox status code.
1768 * @param pVCpu The cross context virtual CPU structure of the calling
1769 * EMT. Can be NULL depending on @a enmTlbFlush.
1770 * @param enmTlbFlush Type of flush.
1771 *
1772 * @remarks Caller is responsible for making sure this function is called only
1773 * when NestedPaging is supported and providing @a enmTlbFlush that is
1774 * supported by the CPU.
1775 * @remarks Can be called with interrupts disabled.
1776 */
1777static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778{
1779 uint64_t au64Descriptor[2];
1780 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781 au64Descriptor[0] = 0;
1782 else
1783 {
1784 Assert(pVCpu);
1785 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786 }
1787 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790 AssertMsg(rc == VINF_SUCCESS,
1791 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793 if ( RT_SUCCESS(rc)
1794 && pVCpu)
1795 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796}
1797
1798
1799/**
1800 * Flushes the TLB using VPID.
1801 *
1802 * @returns VBox status code.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmTlbFlush.
1805 * @param enmTlbFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmTlbFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812{
1813 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834 if ( RT_SUCCESS(rc)
1835 && pVCpu)
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 NOREF(rc);
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVCpu The cross context virtual CPU structure.
1847 * @param GCVirt Guest virtual address of the page to invalidate.
1848 */
1849VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850{
1851 AssertPtr(pVCpu);
1852 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 if (!fFlushPending)
1856 {
1857 /*
1858 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859 * the EPT case. See @bugref{6043} and @bugref{6177}.
1860 *
1861 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862 * as this function maybe called in a loop with individual addresses.
1863 */
1864 PVM pVM = pVCpu->CTX_SUFF(pVM);
1865 if (pVM->hm.s.vmx.fVpid)
1866 {
1867 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870 /*
1871 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872 * where executing INVVPID outside 64-bit mode does not flush translations of
1873 * 64-bit linear addresses, see @bugref{6208#c72}.
1874 */
1875 if (RT_HI_U32(GCVirt))
1876 fVpidFlush = false;
1877#endif
1878
1879 if (fVpidFlush)
1880 {
1881 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883 }
1884 else
1885 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886 }
1887 else if (pVM->hm.s.fNestedPaging)
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897 * case where neither EPT nor VPID is supported by the CPU.
1898 *
1899 * @param pVCpu The cross context virtual CPU structure.
1900 * @param pCpu Pointer to the global HM struct.
1901 *
1902 * @remarks Called with interrupts disabled.
1903 */
1904static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905{
1906 AssertPtr(pVCpu);
1907 AssertPtr(pCpu);
1908
1909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914 pVCpu->hm.s.fForceTLBFlush = false;
1915 return;
1916}
1917
1918
1919/**
1920 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921 *
1922 * @param pVCpu The cross context virtual CPU structure.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926 * nomenclature. The reason is, to avoid confusion in compare statements
1927 * since the host-CPU copies are named "ASID".
1928 *
1929 * @remarks Called with interrupts disabled.
1930 */
1931static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932{
1933#ifdef VBOX_WITH_STATISTICS
1934 bool fTlbFlushed = false;
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937 if (!fTlbFlushed) \
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939 } while (0)
1940#else
1941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943#endif
1944
1945 AssertPtr(pCpu);
1946 AssertPtr(pVCpu);
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949 PVM pVM = pVCpu->CTX_SUFF(pVM);
1950 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954 /*
1955 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958 * cannot reuse the current ASID anymore.
1959 */
1960 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962 {
1963 ++pCpu->uCurrentAsid;
1964 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965 {
1966 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969 }
1970
1971 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975 /*
1976 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978 */
1979 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981 HMVMX_SET_TAGGED_TLB_FLUSHED();
1982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1983 }
1984
1985 /* Check for explicit TLB flushes. */
1986 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1987 {
1988 /*
1989 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1990 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1991 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1992 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1993 * mappings, see @bugref{6568}.
1994 *
1995 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1996 */
1997 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1998 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1999 HMVMX_SET_TAGGED_TLB_FLUSHED();
2000 }
2001
2002 pVCpu->hm.s.fForceTLBFlush = false;
2003 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2004
2005 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2006 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2007 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2008 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2009 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2010 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2011 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2012 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2013 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2014
2015 /* Update VMCS with the VPID. */
2016 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2017 AssertRC(rc);
2018
2019#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2020}
2021
2022
2023/**
2024 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2025 *
2026 * @returns VBox status code.
2027 * @param pVCpu The cross context virtual CPU structure.
2028 * @param pCpu Pointer to the global HM CPU struct.
2029 *
2030 * @remarks Called with interrupts disabled.
2031 */
2032static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2033{
2034 AssertPtr(pVCpu);
2035 AssertPtr(pCpu);
2036 Assert(pCpu->idCpu != NIL_RTCPUID);
2037 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2038 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2039
2040 /*
2041 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2042 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2043 */
2044 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2045 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2049 }
2050
2051 /* Check for explicit TLB flushes. */
2052 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2053 {
2054 pVCpu->hm.s.fForceTLBFlush = true;
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2056 }
2057
2058 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2059 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2060
2061 if (pVCpu->hm.s.fForceTLBFlush)
2062 {
2063 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2064 pVCpu->hm.s.fForceTLBFlush = false;
2065 }
2066}
2067
2068
2069/**
2070 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2071 *
2072 * @returns VBox status code.
2073 * @param pVCpu The cross context virtual CPU structure.
2074 * @param pCpu Pointer to the global HM CPU struct.
2075 *
2076 * @remarks Called with interrupts disabled.
2077 */
2078static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2079{
2080 AssertPtr(pVCpu);
2081 AssertPtr(pCpu);
2082 Assert(pCpu->idCpu != NIL_RTCPUID);
2083 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2084 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2085
2086 /*
2087 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2088 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2089 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2090 * cannot reuse the current ASID anymore.
2091 */
2092 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2093 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2094 {
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2097 }
2098
2099 /* Check for explicit TLB flushes. */
2100 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2101 {
2102 /*
2103 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2104 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2105 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2106 * include fExplicitFlush's too) - an obscure corner case.
2107 */
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2110 }
2111
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 ++pCpu->uCurrentAsid;
2117 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2118 {
2119 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2120 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2121 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2122 }
2123
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2126 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2127 if (pCpu->fFlushAsidBeforeUse)
2128 {
2129 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2130 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2131 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2132 {
2133 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2134 pCpu->fFlushAsidBeforeUse = false;
2135 }
2136 else
2137 {
2138 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2139 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2140 }
2141 }
2142 }
2143
2144 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2145 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2146 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2147 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2148 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2149 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2150 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2151
2152 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2153 AssertRC(rc);
2154}
2155
2156
2157/**
2158 * Flushes the guest TLB entry based on CPU capabilities.
2159 *
2160 * @param pVCpu The cross context virtual CPU structure.
2161 * @param pCpu Pointer to the global HM CPU struct.
2162 */
2163DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2164{
2165#ifdef HMVMX_ALWAYS_FLUSH_TLB
2166 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2167#endif
2168 PVM pVM = pVCpu->CTX_SUFF(pVM);
2169 switch (pVM->hm.s.vmx.enmTlbFlushType)
2170 {
2171 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2174 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2175 default:
2176 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2177 break;
2178 }
2179 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2180}
2181
2182
2183/**
2184 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2185 * TLB entries from the host TLB before VM-entry.
2186 *
2187 * @returns VBox status code.
2188 * @param pVM The cross context VM structure.
2189 */
2190static int hmR0VmxSetupTaggedTlb(PVM pVM)
2191{
2192 /*
2193 * Determine optimal flush type for Nested Paging.
2194 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2195 * guest execution (see hmR3InitFinalizeR0()).
2196 */
2197 if (pVM->hm.s.fNestedPaging)
2198 {
2199 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2200 {
2201 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2203 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2204 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2205 else
2206 {
2207 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2208 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2209 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2210 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211 }
2212
2213 /* Make sure the write-back cacheable memory type for EPT is supported. */
2214 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2215 {
2216 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2217 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219 }
2220
2221 /* EPT requires a page-walk length of 4. */
2222 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2223 {
2224 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2225 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2226 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227 }
2228 }
2229 else
2230 {
2231 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2232 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2233 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2234 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235 }
2236 }
2237
2238 /*
2239 * Determine optimal flush type for VPID.
2240 */
2241 if (pVM->hm.s.vmx.fVpid)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2253 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2255 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2256 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2257 pVM->hm.s.vmx.fVpid = false;
2258 }
2259 }
2260 else
2261 {
2262 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2263 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2264 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2265 pVM->hm.s.vmx.fVpid = false;
2266 }
2267 }
2268
2269 /*
2270 * Setup the handler for flushing tagged-TLBs.
2271 */
2272 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2274 else if (pVM->hm.s.fNestedPaging)
2275 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2276 else if (pVM->hm.s.vmx.fVpid)
2277 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2278 else
2279 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2280 return VINF_SUCCESS;
2281}
2282
2283
2284/**
2285 * Sets up pin-based VM-execution controls in the VMCS.
2286 *
2287 * @returns VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure.
2289 *
2290 * @remarks We don't really care about optimizing vmwrites here as it's done only
2291 * once per VM and hence we don't care about VMCS-field cache comparisons.
2292 */
2293static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2294{
2295 PVM pVM = pVCpu->CTX_SUFF(pVM);
2296 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2298
2299 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2300 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2301
2302 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2303 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2304
2305 /* Enable the VMX preemption timer. */
2306 if (pVM->hm.s.vmx.fUsePreemptTimer)
2307 {
2308 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2309 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2310 }
2311
2312#if 0
2313 /* Enable posted-interrupt processing. */
2314 if (pVM->hm.s.fPostedIntrs)
2315 {
2316 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2317 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2318 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2319 }
2320#endif
2321
2322 if ((fVal & fZap) != fVal)
2323 {
2324 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2325 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2328 }
2329
2330 /* Commit it to the VMCS and update our cache. */
2331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2332 AssertRCReturn(rc, rc);
2333 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2334
2335 return VINF_SUCCESS;
2336}
2337
2338
2339/**
2340 * Sets up secondary processor-based VM-execution controls in the VMCS.
2341 *
2342 * @returns VBox status code.
2343 * @param pVCpu The cross context virtual CPU structure.
2344 *
2345 * @remarks We don't really care about optimizing vmwrites here as it's done only
2346 * once per VM and hence we don't care about VMCS-field cache comparisons.
2347 */
2348static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2349{
2350 PVM pVM = pVCpu->CTX_SUFF(pVM);
2351 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354 /* WBINVD causes a VM-exit. */
2355 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2357
2358 /* Enable EPT (aka nested-paging). */
2359 if (pVM->hm.s.fNestedPaging)
2360 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2361
2362 /*
2363 * Enable the INVPCID instruction if supported by the hardware and we expose
2364 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2365 */
2366 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2367 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2369
2370 /* Enable VPID. */
2371 if (pVM->hm.s.vmx.fVpid)
2372 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2373
2374 /* Enable Unrestricted guest execution. */
2375 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2376 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2377
2378#if 0
2379 if (pVM->hm.s.fVirtApicRegs)
2380 {
2381 /* Enable APIC-register virtualization. */
2382 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2384
2385 /* Enable virtual-interrupt delivery. */
2386 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2387 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2388 }
2389#endif
2390
2391 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2392 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2393 * done dynamically. */
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2395 {
2396 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2397 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2399 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2400 AssertRCReturn(rc, rc);
2401 }
2402
2403 /* Enable RDTSCP. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2405 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2406
2407 /* Enable Pause-Loop exiting. */
2408 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2409 && pVM->hm.s.vmx.cPleGapTicks
2410 && pVM->hm.s.vmx.cPleWindowTicks)
2411 {
2412 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2413
2414 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2415 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2416 AssertRCReturn(rc, rc);
2417 }
2418
2419 if ((fVal & fZap) != fVal)
2420 {
2421 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2422 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2423 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2424 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2425 }
2426
2427 /* Commit it to the VMCS and update our cache. */
2428 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2429 AssertRCReturn(rc, rc);
2430 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2431
2432 return VINF_SUCCESS;
2433}
2434
2435
2436/**
2437 * Sets up processor-based VM-execution controls in the VMCS.
2438 *
2439 * @returns VBox status code.
2440 * @param pVCpu The cross context virtual CPU structure.
2441 *
2442 * @remarks We don't really care about optimizing vmwrites here as it's done only
2443 * once per VM and hence we don't care about VMCS-field cache comparisons.
2444 */
2445static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2446{
2447 PVM pVM = pVCpu->CTX_SUFF(pVM);
2448 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2449 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2452 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2453 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2454 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2455 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2456 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2458
2459 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2460 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2461 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2462 {
2463 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2464 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2465 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2466 }
2467
2468 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2469 if (!pVM->hm.s.fNestedPaging)
2470 {
2471 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2473 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2474 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2475 }
2476
2477 /* Use TPR shadowing if supported by the CPU. */
2478 if ( PDMHasApic(pVM)
2479 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480 {
2481 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2484 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2485 AssertRCReturn(rc, rc);
2486
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2488 /* CR8 writes cause a VM-exit based on TPR threshold. */
2489 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2490 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2491 }
2492 else
2493 {
2494 /*
2495 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2496 * Set this control only for 64-bit guests.
2497 */
2498 if (pVM->hm.s.fAllow64BitGuests)
2499 {
2500 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2501 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2502 }
2503 }
2504
2505 /* Use MSR-bitmaps if supported by the CPU. */
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2507 {
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2509
2510 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2512 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2513 AssertRCReturn(rc, rc);
2514
2515 /*
2516 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2517 * automatically using dedicated fields in the VMCS.
2518 */
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524#if HC_ARCH_BITS == 64
2525 /*
2526 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2527 */
2528 if (pVM->hm.s.fAllow64BitGuests)
2529 {
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534 }
2535#endif
2536 /*
2537 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2538 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2539 */
2540 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2541 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542
2543 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2544 }
2545
2546 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2547 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2549
2550 if ((fVal & fZap) != fVal)
2551 {
2552 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2553 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Commit it to the VMCS and update our cache. */
2559 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2560 AssertRCReturn(rc, rc);
2561 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2562
2563 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2564 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2565 return hmR0VmxSetupProcCtls2(pVCpu);
2566
2567 /* Sanity check, should not really happen. */
2568 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569 {
2570 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2571 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573 }
2574
2575 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2576 return VINF_SUCCESS;
2577}
2578
2579
2580/**
2581 * Sets up miscellaneous (everything other than Pin & Processor-based
2582 * VM-execution) control fields in the VMCS.
2583 *
2584 * @returns VBox status code.
2585 * @param pVCpu The cross context virtual CPU structure.
2586 */
2587static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2588{
2589 AssertPtr(pVCpu);
2590
2591 int rc = VERR_GENERAL_FAILURE;
2592
2593 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594#if 0
2595 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2596 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2598
2599 /*
2600 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2601 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2602 * We thus use the exception bitmap to control it rather than use both.
2603 */
2604 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2606
2607 /* All IO & IOIO instructions cause VM-exits. */
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2609 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2610
2611 /* Initialize the MSR-bitmap area. */
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2614 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 /* Setup MSR auto-load/store area. */
2619 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623 AssertRCReturn(rc, rc);
2624
2625 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2626 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2628 AssertRCReturn(rc, rc);
2629
2630 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2632 AssertRCReturn(rc, rc);
2633
2634 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2635#if 0
2636 /* Setup debug controls */
2637 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2638 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2639 AssertRCReturn(rc, rc);
2640#endif
2641
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2648 *
2649 * We shall setup those exception intercepts that don't change during the
2650 * lifetime of the VM here. The rest are done dynamically while loading the
2651 * guest state.
2652 *
2653 * @returns VBox status code.
2654 * @param pVCpu The cross context virtual CPU structure.
2655 */
2656static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2657{
2658 AssertPtr(pVCpu);
2659
2660 uint32_t uXcptBitmap;
2661
2662 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2663 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2664
2665 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2666 and writes, and because recursive #DBs can cause the CPU hang, we must always
2667 intercept #DB. */
2668 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2669
2670 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2671 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2672 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2673
2674 /* Commit it to the VMCS. */
2675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2676 AssertRCReturn(rc, rc);
2677
2678 /* Update our cache of the exception bitmap. */
2679 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2740 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2741 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2742 */
2743 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2744 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2745 || !pVM->hm.s.vmx.pRealModeTSS))
2746 {
2747 LogRelFunc(("Invalid real-on-v86 state.\n"));
2748 return VERR_INTERNAL_ERROR;
2749 }
2750
2751 /* Initialize these always, see hmR3InitFinalizeR0().*/
2752 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2753 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2754
2755 /* Setup the tagged-TLB flush handlers. */
2756 int rc = hmR0VmxSetupTaggedTlb(pVM);
2757 if (RT_FAILURE(rc))
2758 {
2759 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2760 return rc;
2761 }
2762
2763 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2764 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2765#if HC_ARCH_BITS == 64
2766 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2768 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2769 {
2770 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2771 }
2772#endif
2773
2774 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2775 RTCCUINTREG uHostCR4 = ASMGetCR4();
2776 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2777 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2778
2779 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2780 {
2781 PVMCPU pVCpu = &pVM->aCpus[i];
2782 AssertPtr(pVCpu);
2783 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2784
2785 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2786 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2787
2788 /* Set revision dword at the beginning of the VMCS structure. */
2789 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2790
2791 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2792 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 /* Load this VMCS as the current VMCS. */
2797 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2799 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801 rc = hmR0VmxSetupPinCtls(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804
2805 rc = hmR0VmxSetupProcCtls(pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2807 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808
2809 rc = hmR0VmxSetupMiscCtls(pVCpu);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 rc = hmR0VmxInitXcptBitmap(pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2816
2817#if HC_ARCH_BITS == 32
2818 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821#endif
2822
2823 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2824 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2826 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2827
2828 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2829
2830 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2831 }
2832
2833 return VINF_SUCCESS;
2834}
2835
2836
2837/**
2838 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2839 * the VMCS.
2840 *
2841 * @returns VBox status code.
2842 */
2843static int hmR0VmxExportHostControlRegs(void)
2844{
2845 RTCCUINTREG uReg = ASMGetCR0();
2846 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847 AssertRCReturn(rc, rc);
2848
2849 uReg = ASMGetCR3();
2850 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2851 AssertRCReturn(rc, rc);
2852
2853 uReg = ASMGetCR4();
2854 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2855 AssertRCReturn(rc, rc);
2856 return rc;
2857}
2858
2859
2860/**
2861 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2862 * the host-state area in the VMCS.
2863 *
2864 * @returns VBox status code.
2865 * @param pVCpu The cross context virtual CPU structure.
2866 */
2867static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2868{
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxExportHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890
2891 /*
2892 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893 * should -not- save the messed up state without restoring the original host-state,
2894 * see @bugref{7240}.
2895 *
2896 * This apparently can happen (most likely the FPU changes), deal with it rather than
2897 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2898 */
2899 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2900 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2901 {
2902 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2903 pVCpu->idCpu));
2904 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2905 }
2906 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2907#else
2908 RT_NOREF(pVCpu);
2909#endif
2910
2911 /*
2912 * Host DS, ES, FS and GS segment registers.
2913 */
2914#if HC_ARCH_BITS == 64
2915 RTSEL uSelDS = ASMGetDS();
2916 RTSEL uSelES = ASMGetES();
2917 RTSEL uSelFS = ASMGetFS();
2918 RTSEL uSelGS = ASMGetGS();
2919#else
2920 RTSEL uSelDS = 0;
2921 RTSEL uSelES = 0;
2922 RTSEL uSelFS = 0;
2923 RTSEL uSelGS = 0;
2924#endif
2925
2926 /*
2927 * Host CS and SS segment registers.
2928 */
2929 RTSEL uSelCS = ASMGetCS();
2930 RTSEL uSelSS = ASMGetSS();
2931
2932 /*
2933 * Host TR segment register.
2934 */
2935 RTSEL uSelTR = ASMGetTR();
2936
2937#if HC_ARCH_BITS == 64
2938 /*
2939 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2940 * gain VM-entry and restore them before we get preempted.
2941 *
2942 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2943 */
2944 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2945 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2946 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2947 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2948# undef VMXLOCAL_ADJUST_HOST_SEG
2949#endif
2950
2951 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2952 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2953 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2954 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2955 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2956 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2957 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2958 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2959 Assert(uSelCS);
2960 Assert(uSelTR);
2961
2962 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2963#if 0
2964 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2965 Assert(uSelSS != 0);
2966#endif
2967
2968 /* Write these host selector fields into the host-state area in the VMCS. */
2969 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2970 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2971#if HC_ARCH_BITS == 64
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2975 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2976#else
2977 NOREF(uSelDS);
2978 NOREF(uSelES);
2979 NOREF(uSelFS);
2980 NOREF(uSelGS);
2981#endif
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2983 AssertRCReturn(rc, rc);
2984
2985 /*
2986 * Host GDTR and IDTR.
2987 */
2988 RTGDTR Gdtr;
2989 RTIDTR Idtr;
2990 RT_ZERO(Gdtr);
2991 RT_ZERO(Idtr);
2992 ASMGetGDTR(&Gdtr);
2993 ASMGetIDTR(&Idtr);
2994 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2995 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2996 AssertRCReturn(rc, rc);
2997
2998#if HC_ARCH_BITS == 64
2999 /*
3000 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3001 * them to the maximum limit (0xffff) on every VM-exit.
3002 */
3003 if (Gdtr.cbGdt != 0xffff)
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3005
3006 /*
3007 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3008 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3009 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3010 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3011 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3012 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3013 * at 0xffff on hosts where we are sure it won't cause trouble.
3014 */
3015# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3016 if (Idtr.cbIdt < 0x0fff)
3017# else
3018 if (Idtr.cbIdt != 0xffff)
3019# endif
3020 {
3021 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3022 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024 }
3025#endif
3026
3027 /*
3028 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3029 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3030 * RPL should be too in most cases.
3031 */
3032 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3033 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3034
3035 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3036#if HC_ARCH_BITS == 64
3037 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3038
3039 /*
3040 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3041 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3042 * restoration if the host has something else. Task switching is not supported in 64-bit
3043 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3044 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3045 *
3046 * [1] See Intel spec. 3.5 "System Descriptor Types".
3047 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3048 */
3049 PVM pVM = pVCpu->CTX_SUFF(pVM);
3050 Assert(pDesc->System.u4Type == 11);
3051 if ( pDesc->System.u16LimitLow != 0x67
3052 || pDesc->System.u4LimitHigh)
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3055 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3056 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3057 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3058 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3059 }
3060
3061 /*
3062 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3063 */
3064 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3065 {
3066 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3067 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3068 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3069 {
3070 /* The GDT is read-only but the writable GDT is available. */
3071 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3072 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3073 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3074 AssertRCReturn(rc, rc);
3075 }
3076 }
3077#else
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return VINF_SUCCESS;
3100}
3101
3102
3103/**
3104 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3105 * host-state area of the VMCS.
3106 *
3107 * Theses MSRs will be automatically restored on the host after every successful
3108 * VM-exit.
3109 *
3110 * @returns VBox status code.
3111 * @param pVCpu The cross context virtual CPU structure.
3112 *
3113 * @remarks No-long-jump zone!!!
3114 */
3115static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3116{
3117 AssertPtr(pVCpu);
3118 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3119
3120 /*
3121 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3122 * rather than swapping them on every VM-entry.
3123 */
3124 hmR0VmxLazySaveHostMsrs(pVCpu);
3125
3126 /*
3127 * Host Sysenter MSRs.
3128 */
3129 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3130#if HC_ARCH_BITS == 32
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3133#else
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3136#endif
3137 AssertRCReturn(rc, rc);
3138
3139 /*
3140 * Host EFER MSR.
3141 *
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3143 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3144 */
3145 PVM pVM = pVCpu->CTX_SUFF(pVM);
3146 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3147 {
3148 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3149 AssertRCReturn(rc, rc);
3150 }
3151
3152 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3153
3154 return VINF_SUCCESS;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3163 * hmR0VMxExportGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 *
3168 * @remarks Requires EFER, CR4.
3169 * @remarks No-long-jump zone!!!
3170 */
3171static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3172{
3173#ifdef HMVMX_ALWAYS_SWAP_EFER
3174 return true;
3175#endif
3176
3177 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3178#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3179 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3180 if (CPUMIsGuestInLongModeEx(pCtx))
3181 return false;
3182#endif
3183
3184 PVM pVM = pVCpu->CTX_SUFF(pVM);
3185 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3186 uint64_t const u64GuestEfer = pCtx->msrEFER;
3187
3188 /*
3189 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3190 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3191 */
3192 if ( CPUMIsGuestInLongModeEx(pCtx)
3193 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3194 {
3195 return true;
3196 }
3197
3198 /*
3199 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3200 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3201 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3202 */
3203 if ( (pCtx->cr4 & X86_CR4_PAE)
3204 && (pCtx->cr0 & X86_CR0_PG)
3205 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3206 {
3207 /* Assert that host is PAE capable. */
3208 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3209 return true;
3210 }
3211
3212 return false;
3213}
3214
3215
3216/**
3217 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3218 *
3219 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3220 * see Intel spec. 24.8.1 "VM-entry controls".
3221 *
3222 * @returns VBox status code.
3223 * @param pVCpu The cross context virtual CPU structure.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3229{
3230 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3231 {
3232 PVM pVM = pVCpu->CTX_SUFF(pVM);
3233 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3234 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3235
3236 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3237 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3238
3239 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3240 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3241 {
3242 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3243 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3244 }
3245 else
3246 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3247
3248 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3249 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3250 && hmR0VmxShouldSwapEferMsr(pVCpu))
3251 {
3252 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3253 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3254 }
3255
3256 /*
3257 * The following should -not- be set (since we're not in SMM mode):
3258 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3259 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3260 */
3261
3262 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3263 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3264
3265 if ((fVal & fZap) != fVal)
3266 {
3267 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3268 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3269 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3271 }
3272
3273 /* Commit it to the VMCS and update our cache. */
3274 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3275 {
3276 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3277 AssertRCReturn(rc, rc);
3278 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3279 }
3280
3281 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3282 }
3283 return VINF_SUCCESS;
3284}
3285
3286
3287/**
3288 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3289 *
3290 * @returns VBox status code.
3291 * @param pVCpu The cross context virtual CPU structure.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3296{
3297 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3298 {
3299 PVM pVM = pVCpu->CTX_SUFF(pVM);
3300 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3301 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3302
3303 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3304 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3305
3306 /*
3307 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3308 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3309 * hmR0VmxExportHostMsrs().
3310 */
3311#if HC_ARCH_BITS == 64
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314#else
3315 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3316 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3317 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3318 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3319 {
3320 /* The switcher returns to long mode, EFER is managed by the switcher. */
3321 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3322 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3323 }
3324 else
3325 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3326#endif
3327
3328 /* If the newer VMCS fields for managing EFER exists, use it. */
3329 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3330 && hmR0VmxShouldSwapEferMsr(pVCpu))
3331 {
3332 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3333 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3334 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3335 }
3336
3337 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3338 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3339
3340 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3341 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3342 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3343
3344 /* Enable saving of the VMX preemption timer value on VM-exit. */
3345 if ( pVM->hm.s.vmx.fUsePreemptTimer
3346 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3347 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3348
3349 if ((fVal & fZap) != fVal)
3350 {
3351 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3352 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3353 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3354 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355 }
3356
3357 /* Commit it to the VMCS and update our cache. */
3358 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3359 {
3360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3361 AssertRCReturn(rc, rc);
3362 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3363 }
3364
3365 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3366 }
3367 return VINF_SUCCESS;
3368}
3369
3370
3371/**
3372 * Sets the TPR threshold in the VMCS.
3373 *
3374 * @returns VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure.
3376 * @param u32TprThreshold The TPR threshold (task-priority class only).
3377 */
3378DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3379{
3380 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3381 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3382 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3383}
3384
3385
3386/**
3387 * Exports the guest APIC TPR state into the VMCS.
3388 *
3389 * @returns VBox status code.
3390 * @param pVCpu The cross context virtual CPU structure.
3391 *
3392 * @remarks No-long-jump zone!!!
3393 */
3394static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3395{
3396 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3397 {
3398 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3399
3400 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3401 && APICIsEnabled(pVCpu))
3402 {
3403 /*
3404 * Setup TPR shadowing.
3405 */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3418 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3419 * priority of the pending interrupt so we can deliver the interrupt. If there
3420 * are no interrupts pending, set threshold to 0 to not cause any
3421 * TPR-below-threshold VM-exits.
3422 */
3423 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3424 uint32_t u32TprThreshold = 0;
3425 if (fPendingIntr)
3426 {
3427 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3428 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3429 const uint8_t u8TprPriority = u8Tpr >> 4;
3430 if (u8PendingPriority <= u8TprPriority)
3431 u32TprThreshold = u8PendingPriority;
3432 }
3433
3434 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3435 AssertRCReturn(rc, rc);
3436 }
3437 }
3438 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3439 }
3440 return VINF_SUCCESS;
3441}
3442
3443
3444/**
3445 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3446 *
3447 * @returns Guest's interruptibility-state.
3448 * @param pVCpu The cross context virtual CPU structure.
3449 *
3450 * @remarks No-long-jump zone!!!
3451 */
3452static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3453{
3454 /*
3455 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3456 */
3457 uint32_t fIntrState = 0;
3458 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459 {
3460 /* If inhibition is active, RIP & RFLAGS should've been accessed
3461 (i.e. read previously from the VMCS or from ring-3). */
3462 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3463#ifdef VBOX_STRICT
3464 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3465 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3466#endif
3467 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pCtx->eflags.Bits.u1IF)
3470 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler
3478 * without executing guest code in VT-x, the flag's condition to be cleared is
3479 * met and thus the cleared state is correct.
3480 */
3481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3482 }
3483 }
3484
3485 /*
3486 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3487 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3488 * setting this would block host-NMIs and IRET will not clear the blocking.
3489 *
3490 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3491 */
3492 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3493 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3494 {
3495 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3496 }
3497
3498 return fIntrState;
3499}
3500
3501
3502/**
3503 * Exports the guest's interruptibility-state into the guest-state area in the
3504 * VMCS.
3505 *
3506 * @returns VBox status code.
3507 * @param pVCpu The cross context virtual CPU structure.
3508 * @param fIntrState The interruptibility-state to set.
3509 */
3510static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3511{
3512 NOREF(pVCpu);
3513 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3514 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3515 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3516}
3517
3518
3519/**
3520 * Exports the exception intercepts required for guest execution in the VMCS.
3521 *
3522 * @returns VBox status code.
3523 * @param pVCpu The cross context virtual CPU structure.
3524 *
3525 * @remarks No-long-jump zone!!!
3526 */
3527static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3528{
3529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3530 {
3531 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3532
3533 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3534 if (pVCpu->hm.s.fGIMTrapXcptUD)
3535 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3536#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3537 else
3538 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3539#endif
3540
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3542 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3543
3544 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3545 {
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3547 AssertRCReturn(rc, rc);
3548 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3549 }
3550
3551 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3552 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3553 }
3554 return VINF_SUCCESS;
3555}
3556
3557
3558/**
3559 * Exports the guest's RIP into the guest-state area in the VMCS.
3560 *
3561 * @returns VBox status code.
3562 * @param pVCpu The cross context virtual CPU structure.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3570 {
3571 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3572
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3574 AssertRCReturn(rc, rc);
3575
3576 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3577 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Exports the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 *
3589 * @remarks No-long-jump zone!!!
3590 */
3591static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3592{
3593 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3594 {
3595 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3596
3597 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3601 }
3602 return VINF_SUCCESS;
3603}
3604
3605
3606/**
3607 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 *
3612 * @remarks No-long-jump zone!!!
3613 */
3614static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3615{
3616 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617 {
3618 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3619
3620 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621 Let us assert it as such and use 32-bit VMWRITE. */
3622 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3623 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3624 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3625 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3626
3627 /*
3628 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3629 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3630 * can run the real-mode guest code under Virtual 8086 mode.
3631 */
3632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633 {
3634 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3637 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639 }
3640
3641 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3642 AssertRCReturn(rc, rc);
3643
3644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3645 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3646 }
3647 return VINF_SUCCESS;
3648}
3649
3650
3651/**
3652 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3653 *
3654 * The guest FPU state is always pre-loaded hence we don't need to bother about
3655 * sharing FPU related CR0 bits between the guest and host.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu The cross context virtual CPU structure.
3659 *
3660 * @remarks No-long-jump zone!!!
3661 */
3662static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3663{
3664 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3665 {
3666 PVM pVM = pVCpu->CTX_SUFF(pVM);
3667 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3668 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3669
3670 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3671 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3672
3673 /*
3674 * Setup VT-x's view of the guest CR0.
3675 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3676 */
3677 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3678 if (pVM->hm.s.fNestedPaging)
3679 {
3680 if (CPUMIsGuestPagingEnabled(pVCpu))
3681 {
3682 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3683 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3684 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3685 }
3686 else
3687 {
3688 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3689 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692
3693 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3694 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3695 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696 }
3697 else
3698 {
3699 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3700 u32GuestCr0 |= X86_CR0_WP;
3701 }
3702
3703 /*
3704 * Guest FPU bits.
3705 *
3706 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3707 * using CR0.TS.
3708 *
3709 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3710 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3711 */
3712 u32GuestCr0 |= X86_CR0_NE;
3713
3714 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3715 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3716
3717 /*
3718 * Update exception intercepts.
3719 */
3720 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3721 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3722 {
3723 Assert(PDMVmmDevHeapIsEnabled(pVM));
3724 Assert(pVM->hm.s.vmx.pRealModeTSS);
3725 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3726 }
3727 else
3728 {
3729 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3730 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3731 if (fInterceptMF)
3732 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3733 }
3734
3735 /* Additional intercepts for debugging, define these yourself explicitly. */
3736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3737 uXcptBitmap |= 0
3738 | RT_BIT(X86_XCPT_BP)
3739 | RT_BIT(X86_XCPT_DE)
3740 | RT_BIT(X86_XCPT_NM)
3741 | RT_BIT(X86_XCPT_TS)
3742 | RT_BIT(X86_XCPT_UD)
3743 | RT_BIT(X86_XCPT_NP)
3744 | RT_BIT(X86_XCPT_SS)
3745 | RT_BIT(X86_XCPT_GP)
3746 | RT_BIT(X86_XCPT_PF)
3747 | RT_BIT(X86_XCPT_MF)
3748 ;
3749#elif defined(HMVMX_ALWAYS_TRAP_PF)
3750 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3751#endif
3752 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3798 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3799 AssertRCReturn(rc, rc);
3800
3801 /* Update our caches. */
3802 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3803 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3804 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3805
3806 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3807
3808 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3809 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3810 }
3811
3812 return VINF_SUCCESS;
3813}
3814
3815
3816/**
3817 * Exports the guest control registers (CR3, CR4) into the guest-state area
3818 * in the VMCS.
3819 *
3820 * @returns VBox strict status code.
3821 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3822 * without unrestricted guest access and the VMMDev is not presently
3823 * mapped (e.g. EFI32).
3824 *
3825 * @param pVCpu The cross context virtual CPU structure.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3845
3846 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3847 if (pVM->hm.s.fNestedPaging)
3848 {
3849 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3850
3851 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3852 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3854 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3855
3856 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3857 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3858 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3859
3860 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3861 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3862 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3863 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3864 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3865 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3866 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867
3868 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3869 AssertRCReturn(rc, rc);
3870
3871 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3872 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3873 || CPUMIsGuestPagingEnabledEx(pCtx))
3874 {
3875 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3876 if (CPUMIsGuestInPAEModeEx(pCtx))
3877 {
3878 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3879 AssertRCReturn(rc, rc);
3880 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3883 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3884 AssertRCReturn(rc, rc);
3885 }
3886
3887 /*
3888 * The guest's view of its CR3 is unblemished with Nested Paging when the
3889 * guest is using paging or we have unrestricted guest execution to handle
3890 * the guest when it's not using paging.
3891 */
3892 GCPhysGuestCR3 = pCtx->cr3;
3893 }
3894 else
3895 {
3896 /*
3897 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3898 * thinks it accesses physical memory directly, we use our identity-mapped
3899 * page table to map guest-linear to guest-physical addresses. EPT takes care
3900 * of translating it to host-physical addresses.
3901 */
3902 RTGCPHYS GCPhys;
3903 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3904
3905 /* We obtain it here every time as the guest could have relocated this PCI region. */
3906 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907 if (RT_SUCCESS(rc))
3908 { /* likely */ }
3909 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3910 {
3911 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3912 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3913 }
3914 else
3915 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3916
3917 GCPhysGuestCR3 = GCPhys;
3918 }
3919
3920 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3921 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922 AssertRCReturn(rc, rc);
3923 }
3924 else
3925 {
3926 /* Non-nested paging case, just use the hypervisor's CR3. */
3927 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3928
3929 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3930 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3931 AssertRCReturn(rc, rc);
3932 }
3933
3934 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3935 }
3936
3937 /*
3938 * Guest CR4.
3939 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3940 */
3941 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3942 {
3943 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3944 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3945 Assert(!RT_HI_U32(pCtx->cr4));
3946
3947 uint32_t u32GuestCr4 = pCtx->cr4;
3948 uint32_t const u32ShadowCr4 = pCtx->cr4;
3949
3950 /*
3951 * Setup VT-x's view of the guest CR4.
3952 *
3953 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3954 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3955 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3956 *
3957 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958 */
3959 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960 {
3961 Assert(pVM->hm.s.vmx.pRealModeTSS);
3962 Assert(PDMVmmDevHeapIsEnabled(pVM));
3963 u32GuestCr4 &= ~X86_CR4_VME;
3964 }
3965
3966 if (pVM->hm.s.fNestedPaging)
3967 {
3968 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3969 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3970 {
3971 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972 u32GuestCr4 |= X86_CR4_PSE;
3973 /* Our identity mapping is a 32-bit page directory. */
3974 u32GuestCr4 &= ~X86_CR4_PAE;
3975 }
3976 /* else use guest CR4.*/
3977 }
3978 else
3979 {
3980 /*
3981 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983 */
3984 switch (pVCpu->hm.s.enmShadowMode)
3985 {
3986 case PGMMODE_REAL: /* Real-mode. */
3987 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988 case PGMMODE_32_BIT: /* 32-bit paging. */
3989 {
3990 u32GuestCr4 &= ~X86_CR4_PAE;
3991 break;
3992 }
3993
3994 case PGMMODE_PAE: /* PAE paging. */
3995 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996 {
3997 u32GuestCr4 |= X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004 break;
4005#endif
4006 default:
4007 AssertFailed();
4008 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009 }
4010 }
4011
4012 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 u32GuestCr4 |= fSetCr4;
4016 u32GuestCr4 &= fZapCr4;
4017
4018 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4019 that would cause a VM-exit. */
4020 uint32_t u32Cr4Mask = X86_CR4_VME
4021 | X86_CR4_PAE
4022 | X86_CR4_PGE
4023 | X86_CR4_PSE
4024 | X86_CR4_VMXE;
4025 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4026 u32Cr4Mask |= X86_CR4_OSXSAVE;
4027 if (pVM->cpum.ro.GuestFeatures.fPcid)
4028 u32Cr4Mask |= X86_CR4_PCIDE;
4029
4030 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4031 into the VMCS and update our cache. */
4032 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4034 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4035 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4036 AssertRCReturn(rc, rc);
4037 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4038
4039 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4040 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4041
4042 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4043
4044 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4045 fZapCr4));
4046 }
4047 return rc;
4048}
4049
4050
4051/**
4052 * Exports the guest debug registers into the guest-state area in the VMCS.
4053 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4054 *
4055 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4056 *
4057 * @returns VBox status code.
4058 * @param pVCpu The cross context virtual CPU structure.
4059 *
4060 * @remarks No-long-jump zone!!!
4061 */
4062static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4063{
4064 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4068 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4069 {
4070 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4072 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4073 }
4074#endif
4075
4076 bool fSteppingDB = false;
4077 bool fInterceptMovDRx = false;
4078 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4079 if (pVCpu->hm.s.fSingleInstruction)
4080 {
4081 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4082 PVM pVM = pVCpu->CTX_SUFF(pVM);
4083 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4084 {
4085 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4086 Assert(fSteppingDB == false);
4087 }
4088 else
4089 {
4090 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4091 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4092 pVCpu->hm.s.fClearTrapFlag = true;
4093 fSteppingDB = true;
4094 }
4095 }
4096
4097 uint32_t u32GuestDr7;
4098 if ( fSteppingDB
4099 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4100 {
4101 /*
4102 * Use the combined guest and host DRx values found in the hypervisor register set
4103 * because the debugger has breakpoints active or someone is single stepping on the
4104 * host side without a monitor trap flag.
4105 *
4106 * Note! DBGF expects a clean DR6 state before executing guest code.
4107 */
4108#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4110 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4111 {
4112 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4113 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4114 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4115 }
4116 else
4117#endif
4118 if (!CPUMIsHyperDebugStateActive(pVCpu))
4119 {
4120 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4121 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4122 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4123 }
4124
4125 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4126 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4127 pVCpu->hm.s.fUsingHyperDR7 = true;
4128 fInterceptMovDRx = true;
4129 }
4130 else
4131 {
4132 /*
4133 * If the guest has enabled debug registers, we need to load them prior to
4134 * executing guest code so they'll trigger at the right time.
4135 */
4136 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4137 {
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4140 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146 }
4147 else
4148#endif
4149 if (!CPUMIsGuestDebugStateActive(pVCpu))
4150 {
4151 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155 }
4156 Assert(!fInterceptMovDRx);
4157 }
4158 /*
4159 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160 * must intercept #DB in order to maintain a correct DR6 guest value, and
4161 * because we need to intercept it to prevent nested #DBs from hanging the
4162 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4163 */
4164#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4165 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4166 && !CPUMIsGuestDebugStateActive(pVCpu))
4167#else
4168 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4169#endif
4170 {
4171 fInterceptMovDRx = true;
4172 }
4173
4174 /* Update DR7 with the actual guest value. */
4175 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4176 pVCpu->hm.s.fUsingHyperDR7 = false;
4177 }
4178
4179 if (fInterceptMovDRx)
4180 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181 else
4182 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4183
4184 /*
4185 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4186 * monitor-trap flag and update our cache.
4187 */
4188 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4189 {
4190 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4191 AssertRCReturn(rc2, rc2);
4192 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4193 }
4194
4195 /*
4196 * Update guest DR7.
4197 */
4198 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4199 AssertRCReturn(rc, rc);
4200
4201 return VINF_SUCCESS;
4202}
4203
4204
4205#ifdef VBOX_STRICT
4206/**
4207 * Strict function to validate segment registers.
4208 *
4209 * @param pVCpu The cross context virtual CPU structure.
4210 *
4211 * @remarks Will import guest CR0 on strict builds during validation of
4212 * segments.
4213 */
4214static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4215{
4216 /*
4217 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4218 *
4219 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4220 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4221 * and doesn't change the guest-context value.
4222 */
4223 PVM pVM = pVCpu->CTX_SUFF(pVM);
4224 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4225 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4226 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4227 && ( !CPUMIsGuestInRealModeEx(pCtx)
4228 && !CPUMIsGuestInV86ModeEx(pCtx)))
4229 {
4230 /* Protected mode checks */
4231 /* CS */
4232 Assert(pCtx->cs.Attr.n.u1Present);
4233 Assert(!(pCtx->cs.Attr.u & 0xf00));
4234 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4235 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4236 || !(pCtx->cs.Attr.n.u1Granularity));
4237 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4238 || (pCtx->cs.Attr.n.u1Granularity));
4239 /* CS cannot be loaded with NULL in protected mode. */
4240 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4241 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4242 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4243 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4244 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4245 else
4246 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4247 /* SS */
4248 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4249 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4250 if ( !(pCtx->cr0 & X86_CR0_PE)
4251 || pCtx->cs.Attr.n.u4Type == 3)
4252 {
4253 Assert(!pCtx->ss.Attr.n.u2Dpl);
4254 }
4255 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4256 {
4257 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4258 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4259 Assert(pCtx->ss.Attr.n.u1Present);
4260 Assert(!(pCtx->ss.Attr.u & 0xf00));
4261 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4262 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4263 || !(pCtx->ss.Attr.n.u1Granularity));
4264 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4265 || (pCtx->ss.Attr.n.u1Granularity));
4266 }
4267 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4268 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4269 {
4270 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4271 Assert(pCtx->ds.Attr.n.u1Present);
4272 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4273 Assert(!(pCtx->ds.Attr.u & 0xf00));
4274 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4275 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4276 || !(pCtx->ds.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4278 || (pCtx->ds.Attr.n.u1Granularity));
4279 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4280 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4281 }
4282 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4285 Assert(pCtx->es.Attr.n.u1Present);
4286 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4287 Assert(!(pCtx->es.Attr.u & 0xf00));
4288 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->es.Attr.n.u1Granularity));
4291 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4292 || (pCtx->es.Attr.n.u1Granularity));
4293 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4294 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4295 }
4296 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4297 {
4298 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4299 Assert(pCtx->fs.Attr.n.u1Present);
4300 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4301 Assert(!(pCtx->fs.Attr.u & 0xf00));
4302 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4303 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4304 || !(pCtx->fs.Attr.n.u1Granularity));
4305 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4306 || (pCtx->fs.Attr.n.u1Granularity));
4307 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4308 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4309 }
4310 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4311 {
4312 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313 Assert(pCtx->gs.Attr.n.u1Present);
4314 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4315 Assert(!(pCtx->gs.Attr.u & 0xf00));
4316 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4317 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4318 || !(pCtx->gs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4320 || (pCtx->gs.Attr.n.u1Granularity));
4321 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323 }
4324 /* 64-bit capable CPUs. */
4325# if HC_ARCH_BITS == 64
4326 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4327 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4328 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4329 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4330# endif
4331 }
4332 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4333 || ( CPUMIsGuestInRealModeEx(pCtx)
4334 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4335 {
4336 /* Real and v86 mode checks. */
4337 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4338 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4339 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4340 {
4341 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4342 }
4343 else
4344 {
4345 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4346 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4347 }
4348
4349 /* CS */
4350 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4351 Assert(pCtx->cs.u32Limit == 0xffff);
4352 Assert(u32CSAttr == 0xf3);
4353 /* SS */
4354 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4355 Assert(pCtx->ss.u32Limit == 0xffff);
4356 Assert(u32SSAttr == 0xf3);
4357 /* DS */
4358 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4359 Assert(pCtx->ds.u32Limit == 0xffff);
4360 Assert(u32DSAttr == 0xf3);
4361 /* ES */
4362 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4363 Assert(pCtx->es.u32Limit == 0xffff);
4364 Assert(u32ESAttr == 0xf3);
4365 /* FS */
4366 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4367 Assert(pCtx->fs.u32Limit == 0xffff);
4368 Assert(u32FSAttr == 0xf3);
4369 /* GS */
4370 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4371 Assert(pCtx->gs.u32Limit == 0xffff);
4372 Assert(u32GSAttr == 0xf3);
4373 /* 64-bit capable CPUs. */
4374# if HC_ARCH_BITS == 64
4375 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4376 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4377 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4378 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4379# endif
4380 }
4381}
4382#endif /* VBOX_STRICT */
4383
4384
4385/**
4386 * Exports a guest segment register into the guest-state area in the VMCS.
4387 *
4388 * @returns VBox status code.
4389 * @param pVCpu The cross context virtual CPU structure.
4390 * @param idxSel Index of the selector in the VMCS.
4391 * @param idxLimit Index of the segment limit in the VMCS.
4392 * @param idxBase Index of the segment base in the VMCS.
4393 * @param idxAccess Index of the access rights of the segment in the VMCS.
4394 * @param pSelReg Pointer to the segment selector.
4395 *
4396 * @remarks No-long-jump zone!!!
4397 */
4398static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4399 PCCPUMSELREG pSelReg)
4400{
4401 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4402 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4403 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4404 AssertRCReturn(rc, rc);
4405
4406 uint32_t u32Access = pSelReg->Attr.u;
4407 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4408 {
4409 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4410 u32Access = 0xf3;
4411 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4412 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4413 }
4414 else
4415 {
4416 /*
4417 * The way to differentiate between whether this is really a null selector or was just
4418 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4419 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4420 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4421 * NULL selectors loaded in protected-mode have their attribute as 0.
4422 */
4423 if (!u32Access)
4424 u32Access = X86DESCATTR_UNUSABLE;
4425 }
4426
4427 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4428 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4429 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4430
4431 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4432 AssertRCReturn(rc, rc);
4433 return rc;
4434}
4435
4436
4437/**
4438 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4439 * into the guest-state area in the VMCS.
4440 *
4441 * @returns VBox status code.
4442 * @param pVCpu The cross context virtual CPU structure.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4453
4454 /*
4455 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4456 */
4457 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4458 {
4459#ifdef VBOX_WITH_REM
4460 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4461 {
4462 Assert(pVM->hm.s.vmx.pRealModeTSS);
4463 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4464 if ( pVCpu->hm.s.vmx.fWasInRealMode
4465 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4466 {
4467 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4468 in real-mode (e.g. OpenBSD 4.0) */
4469 REMFlushTBs(pVM);
4470 Log4Func(("Switch to protected mode detected!\n"));
4471 pVCpu->hm.s.vmx.fWasInRealMode = false;
4472 }
4473 }
4474#endif
4475 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4476 {
4477 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4478 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4479 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4480 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4481 AssertRCReturn(rc, rc);
4482 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4483 }
4484
4485 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4486 {
4487 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4488 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4489 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4490 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4491 AssertRCReturn(rc, rc);
4492 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4493 }
4494
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4496 {
4497 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4500 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4501 AssertRCReturn(rc, rc);
4502 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4503 }
4504
4505 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4506 {
4507 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4508 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4509 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4510 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4511 AssertRCReturn(rc, rc);
4512 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4513 }
4514
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4516 {
4517 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4518 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4519 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4520 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4521 AssertRCReturn(rc, rc);
4522 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4523 }
4524
4525 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4526 {
4527 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4528 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4529 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4530 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4531 AssertRCReturn(rc, rc);
4532 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4533 }
4534
4535#ifdef VBOX_STRICT
4536 hmR0VmxValidateSegmentRegs(pVCpu);
4537#endif
4538
4539 /* Update the exit history entry with the correct CS.BASE + RIP. */
4540 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4541 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
4542
4543 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4544 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4545 }
4546
4547 /*
4548 * Guest TR.
4549 */
4550 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4551 {
4552 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4553
4554 /*
4555 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4556 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4557 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4558 */
4559 uint16_t u16Sel = 0;
4560 uint32_t u32Limit = 0;
4561 uint64_t u64Base = 0;
4562 uint32_t u32AccessRights = 0;
4563
4564 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4565 {
4566 u16Sel = pCtx->tr.Sel;
4567 u32Limit = pCtx->tr.u32Limit;
4568 u64Base = pCtx->tr.u64Base;
4569 u32AccessRights = pCtx->tr.Attr.u;
4570 }
4571 else
4572 {
4573 Assert(pVM->hm.s.vmx.pRealModeTSS);
4574 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4575
4576 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4577 RTGCPHYS GCPhys;
4578 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4579 AssertRCReturn(rc, rc);
4580
4581 X86DESCATTR DescAttr;
4582 DescAttr.u = 0;
4583 DescAttr.n.u1Present = 1;
4584 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4585
4586 u16Sel = 0;
4587 u32Limit = HM_VTX_TSS_SIZE;
4588 u64Base = GCPhys; /* in real-mode phys = virt. */
4589 u32AccessRights = DescAttr.u;
4590 }
4591
4592 /* Validate. */
4593 Assert(!(u16Sel & RT_BIT(2)));
4594 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4595 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4596 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4597 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4598 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4599 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4600 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4601 Assert( (u32Limit & 0xfff) == 0xfff
4602 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4603 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4604 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4605
4606 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4607 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4609 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4610 AssertRCReturn(rc, rc);
4611
4612 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4613 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4614 }
4615
4616 /*
4617 * Guest GDTR.
4618 */
4619 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4620 {
4621 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4622
4623 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4624 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4625 AssertRCReturn(rc, rc);
4626
4627 /* Validate. */
4628 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4629
4630 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4631 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4632 }
4633
4634 /*
4635 * Guest LDTR.
4636 */
4637 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4638 {
4639 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4640
4641 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4642 uint32_t u32Access = 0;
4643 if (!pCtx->ldtr.Attr.u)
4644 u32Access = X86DESCATTR_UNUSABLE;
4645 else
4646 u32Access = pCtx->ldtr.Attr.u;
4647
4648 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4649 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4650 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4651 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4652 AssertRCReturn(rc, rc);
4653
4654 /* Validate. */
4655 if (!(u32Access & X86DESCATTR_UNUSABLE))
4656 {
4657 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4658 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4659 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4660 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4661 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4662 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4663 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4664 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4665 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4666 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4667 }
4668
4669 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4670 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4671 }
4672
4673 /*
4674 * Guest IDTR.
4675 */
4676 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4677 {
4678 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4679
4680 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4681 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4682 AssertRCReturn(rc, rc);
4683
4684 /* Validate. */
4685 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4686
4687 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4688 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4689 }
4690
4691 return VINF_SUCCESS;
4692}
4693
4694
4695/**
4696 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4697 * areas.
4698 *
4699 * These MSRs will automatically be loaded to the host CPU on every successful
4700 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4701 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4702 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4703 *
4704 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4705 *
4706 * @returns VBox status code.
4707 * @param pVCpu The cross context virtual CPU structure.
4708 *
4709 * @remarks No-long-jump zone!!!
4710 */
4711static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4712{
4713 AssertPtr(pVCpu);
4714 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4715
4716 /*
4717 * MSRs that we use the auto-load/store MSR area in the VMCS.
4718 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4719 */
4720 PVM pVM = pVCpu->CTX_SUFF(pVM);
4721 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4722 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4723 {
4724 if (pVM->hm.s.fAllow64BitGuests)
4725 {
4726#if HC_ARCH_BITS == 32
4727 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4728
4729 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4730 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4731 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4732 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4733 AssertRCReturn(rc, rc);
4734# ifdef LOG_ENABLED
4735 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4736 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4737 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4738# endif
4739#endif
4740 }
4741 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4742 }
4743
4744 /*
4745 * Guest Sysenter MSRs.
4746 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4747 * VM-exits on WRMSRs for these MSRs.
4748 */
4749 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4750 {
4751 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4752
4753 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4754 {
4755 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4756 AssertRCReturn(rc, rc);
4757 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4758 }
4759
4760 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4761 {
4762 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4763 AssertRCReturn(rc, rc);
4764 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4765 }
4766
4767 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4768 {
4769 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4770 AssertRCReturn(rc, rc);
4771 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4772 }
4773 }
4774
4775 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4776 {
4777 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4778
4779 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4780 {
4781 /*
4782 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4783 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4784 */
4785 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4786 {
4787 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4788 AssertRCReturn(rc,rc);
4789 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4790 }
4791 else
4792 {
4793 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4794 NULL /* pfAddedAndUpdated */);
4795 AssertRCReturn(rc, rc);
4796
4797 /* We need to intercept reads too, see @bugref{7386#c16}. */
4798 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4799 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4800 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4801 pVCpu->hm.s.vmx.cMsrs));
4802 }
4803 }
4804 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4805 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4806 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4807 }
4808
4809 return VINF_SUCCESS;
4810}
4811
4812
4813#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4814/**
4815 * Check if guest state allows safe use of 32-bit switcher again.
4816 *
4817 * Segment bases and protected mode structures must be 32-bit addressable
4818 * because the 32-bit switcher will ignore high dword when writing these VMCS
4819 * fields. See @bugref{8432} for details.
4820 *
4821 * @returns true if safe, false if must continue to use the 64-bit switcher.
4822 * @param pCtx Pointer to the guest-CPU context.
4823 *
4824 * @remarks No-long-jump zone!!!
4825 */
4826static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4827{
4828 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4835 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4836 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4837 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4838
4839 /* All good, bases are 32-bit. */
4840 return true;
4841}
4842#endif
4843
4844
4845/**
4846 * Selects up the appropriate function to run guest code.
4847 *
4848 * @returns VBox status code.
4849 * @param pVCpu The cross context virtual CPU structure.
4850 *
4851 * @remarks No-long-jump zone!!!
4852 */
4853static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4854{
4855 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4856 if (CPUMIsGuestInLongModeEx(pCtx))
4857 {
4858#ifndef VBOX_ENABLE_64_BITS_GUESTS
4859 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4860#endif
4861 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4862#if HC_ARCH_BITS == 32
4863 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4864 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4865 {
4866#ifdef VBOX_STRICT
4867 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4868 {
4869 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4870 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4871 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4872 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4873 | HM_CHANGED_VMX_ENTRY_CTLS
4874 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4875 }
4876#endif
4877 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4878
4879 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4880 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4881 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4882 Log4Func(("Selected 64-bit switcher\n"));
4883 }
4884#else
4885 /* 64-bit host. */
4886 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4887#endif
4888 }
4889 else
4890 {
4891 /* Guest is not in long mode, use the 32-bit handler. */
4892#if HC_ARCH_BITS == 32
4893 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4894 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4895 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4896 {
4897# ifdef VBOX_STRICT
4898 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4899 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4900 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4901 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4902 | HM_CHANGED_VMX_ENTRY_CTLS
4903 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4904# endif
4905 }
4906# ifdef VBOX_ENABLE_64_BITS_GUESTS
4907 /*
4908 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4909 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4910 * switcher flag because now we know the guest is in a sane state where it's safe
4911 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4912 * the much faster 32-bit switcher again.
4913 */
4914 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4915 {
4916 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4917 Log4Func(("Selected 32-bit switcher\n"));
4918 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4919 }
4920 else
4921 {
4922 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4923 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4924 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4925 {
4926 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4927 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4928 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4929 | HM_CHANGED_VMX_ENTRY_CTLS
4930 | HM_CHANGED_VMX_EXIT_CTLS
4931 | HM_CHANGED_HOST_CONTEXT);
4932 Log4Func(("Selected 32-bit switcher (safe)\n"));
4933 }
4934 }
4935# else
4936 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4937# endif
4938#else
4939 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4940#endif
4941 }
4942 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4943 return VINF_SUCCESS;
4944}
4945
4946
4947/**
4948 * Wrapper for running the guest code in VT-x.
4949 *
4950 * @returns VBox status code, no informational status codes.
4951 * @param pVCpu The cross context virtual CPU structure.
4952 *
4953 * @remarks No-long-jump zone!!!
4954 */
4955DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4956{
4957 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4958 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4959 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4960
4961 /*
4962 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4963 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4964 * callee-saved and thus the need for this XMM wrapper.
4965 *
4966 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4967 */
4968 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4969 /** @todo Add stats for resume vs launch. */
4970 PVM pVM = pVCpu->CTX_SUFF(pVM);
4971#ifdef VBOX_WITH_KERNEL_USING_XMM
4972 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4973#else
4974 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4975#endif
4976 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4977 return rc;
4978}
4979
4980
4981/**
4982 * Reports world-switch error and dumps some useful debug info.
4983 *
4984 * @param pVCpu The cross context virtual CPU structure.
4985 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4986 * @param pVmxTransient Pointer to the VMX transient structure (only
4987 * exitReason updated).
4988 */
4989static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4990{
4991 Assert(pVCpu);
4992 Assert(pVmxTransient);
4993 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4994
4995 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4996 switch (rcVMRun)
4997 {
4998 case VERR_VMX_INVALID_VMXON_PTR:
4999 AssertFailed();
5000 break;
5001 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5002 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5003 {
5004 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5005 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5006 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5007 AssertRC(rc);
5008
5009 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5010 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5011 Cannot do it here as we may have been long preempted. */
5012
5013#ifdef VBOX_STRICT
5014 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5015 pVmxTransient->uExitReason));
5016 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5017 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5018 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5019 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5020 else
5021 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5022 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5023 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5024
5025 /* VMX control bits. */
5026 uint32_t u32Val;
5027 uint64_t u64Val;
5028 RTHCUINTREG uHCReg;
5029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5031 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5032 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5033 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5034 {
5035 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5036 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5037 }
5038 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5039 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5040 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5041 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5042 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5043 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5044 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5046 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5047 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5048 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5049 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5050 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5051 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5056 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5057 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5060 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5061 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5062 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5063 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5064 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5065 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5066 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5067 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5068 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5069 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5070 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5071 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5072 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5073 {
5074 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5075 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5076 }
5077
5078 /* Guest bits. */
5079 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5080 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5081 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5082 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5083 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5084 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5085 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5086 {
5087 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5088 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5089 }
5090
5091 /* Host bits. */
5092 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5093 Log4(("Host CR0 %#RHr\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5095 Log4(("Host CR3 %#RHr\n", uHCReg));
5096 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5097 Log4(("Host CR4 %#RHr\n", uHCReg));
5098
5099 RTGDTR HostGdtr;
5100 PCX86DESCHC pDesc;
5101 ASMGetGDTR(&HostGdtr);
5102 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5103 Log4(("Host CS %#08x\n", u32Val));
5104 if (u32Val < HostGdtr.cbGdt)
5105 {
5106 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5107 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5108 }
5109
5110 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5111 Log4(("Host DS %#08x\n", u32Val));
5112 if (u32Val < HostGdtr.cbGdt)
5113 {
5114 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5115 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5116 }
5117
5118 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5119 Log4(("Host ES %#08x\n", u32Val));
5120 if (u32Val < HostGdtr.cbGdt)
5121 {
5122 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5123 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5124 }
5125
5126 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5127 Log4(("Host FS %#08x\n", u32Val));
5128 if (u32Val < HostGdtr.cbGdt)
5129 {
5130 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5132 }
5133
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host GS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host SS %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5148 }
5149
5150 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5151 Log4(("Host TR %#08x\n", u32Val));
5152 if (u32Val < HostGdtr.cbGdt)
5153 {
5154 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5155 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5156 }
5157
5158 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5159 Log4(("Host TR Base %#RHv\n", uHCReg));
5160 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5161 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5162 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5163 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5164 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5165 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5166 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5167 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5168 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5169 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5170 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5171 Log4(("Host RSP %#RHv\n", uHCReg));
5172 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5173 Log4(("Host RIP %#RHv\n", uHCReg));
5174# if HC_ARCH_BITS == 64
5175 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5176 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5177 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5178 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5179 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5180 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5181# endif
5182#endif /* VBOX_STRICT */
5183 break;
5184 }
5185
5186 default:
5187 /* Impossible */
5188 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5189 break;
5190 }
5191}
5192
5193
5194#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5195#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5196# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5197#endif
5198#ifdef VBOX_STRICT
5199static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5200{
5201 switch (idxField)
5202 {
5203 case VMX_VMCS_GUEST_RIP:
5204 case VMX_VMCS_GUEST_RSP:
5205 case VMX_VMCS_GUEST_SYSENTER_EIP:
5206 case VMX_VMCS_GUEST_SYSENTER_ESP:
5207 case VMX_VMCS_GUEST_GDTR_BASE:
5208 case VMX_VMCS_GUEST_IDTR_BASE:
5209 case VMX_VMCS_GUEST_CS_BASE:
5210 case VMX_VMCS_GUEST_DS_BASE:
5211 case VMX_VMCS_GUEST_ES_BASE:
5212 case VMX_VMCS_GUEST_FS_BASE:
5213 case VMX_VMCS_GUEST_GS_BASE:
5214 case VMX_VMCS_GUEST_SS_BASE:
5215 case VMX_VMCS_GUEST_LDTR_BASE:
5216 case VMX_VMCS_GUEST_TR_BASE:
5217 case VMX_VMCS_GUEST_CR3:
5218 return true;
5219 }
5220 return false;
5221}
5222
5223static bool hmR0VmxIsValidReadField(uint32_t idxField)
5224{
5225 switch (idxField)
5226 {
5227 /* Read-only fields. */
5228 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5229 return true;
5230 }
5231 /* Remaining readable fields should also be writable. */
5232 return hmR0VmxIsValidWriteField(idxField);
5233}
5234#endif /* VBOX_STRICT */
5235
5236
5237/**
5238 * Executes the specified handler in 64-bit mode.
5239 *
5240 * @returns VBox status code (no informational status codes).
5241 * @param pVCpu The cross context virtual CPU structure.
5242 * @param enmOp The operation to perform.
5243 * @param cParams Number of parameters.
5244 * @param paParam Array of 32-bit parameters.
5245 */
5246VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5247{
5248 PVM pVM = pVCpu->CTX_SUFF(pVM);
5249 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5250 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5251 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5252 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5253
5254#ifdef VBOX_STRICT
5255 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5256 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5257
5258 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5259 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5260#endif
5261
5262 /* Disable interrupts. */
5263 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5264
5265#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5266 RTCPUID idHostCpu = RTMpCpuId();
5267 CPUMR0SetLApic(pVCpu, idHostCpu);
5268#endif
5269
5270 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5271 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5272
5273 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5274 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5275 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5276
5277 /* Leave VMX Root Mode. */
5278 VMXDisable();
5279
5280 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5281
5282 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5283 CPUMSetHyperEIP(pVCpu, enmOp);
5284 for (int i = (int)cParams - 1; i >= 0; i--)
5285 CPUMPushHyper(pVCpu, paParam[i]);
5286
5287 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5288
5289 /* Call the switcher. */
5290 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5291 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5292
5293 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5294 /* Make sure the VMX instructions don't cause #UD faults. */
5295 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5296
5297 /* Re-enter VMX Root Mode */
5298 int rc2 = VMXEnable(HCPhysCpuPage);
5299 if (RT_FAILURE(rc2))
5300 {
5301 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5302 ASMSetFlags(fOldEFlags);
5303 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5304 return rc2;
5305 }
5306
5307 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5308 AssertRC(rc2);
5309 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5310 Assert(!(ASMGetFlags() & X86_EFL_IF));
5311 ASMSetFlags(fOldEFlags);
5312 return rc;
5313}
5314
5315
5316/**
5317 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5318 * supporting 64-bit guests.
5319 *
5320 * @returns VBox status code.
5321 * @param fResume Whether to VMLAUNCH or VMRESUME.
5322 * @param pCtx Pointer to the guest-CPU context.
5323 * @param pCache Pointer to the VMCS cache.
5324 * @param pVM The cross context VM structure.
5325 * @param pVCpu The cross context virtual CPU structure.
5326 */
5327DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5328{
5329 NOREF(fResume);
5330
5331 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5332 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5333
5334#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5335 pCache->uPos = 1;
5336 pCache->interPD = PGMGetInterPaeCR3(pVM);
5337 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5338#endif
5339
5340#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5341 pCache->TestIn.HCPhysCpuPage = 0;
5342 pCache->TestIn.HCPhysVmcs = 0;
5343 pCache->TestIn.pCache = 0;
5344 pCache->TestOut.HCPhysVmcs = 0;
5345 pCache->TestOut.pCache = 0;
5346 pCache->TestOut.pCtx = 0;
5347 pCache->TestOut.eflags = 0;
5348#else
5349 NOREF(pCache);
5350#endif
5351
5352 uint32_t aParam[10];
5353 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5354 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5355 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5356 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5357 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5358 aParam[5] = 0;
5359 aParam[6] = VM_RC_ADDR(pVM, pVM);
5360 aParam[7] = 0;
5361 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5362 aParam[9] = 0;
5363
5364#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5365 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5366 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5367#endif
5368 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5369
5370#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5371 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5372 Assert(pCtx->dr[4] == 10);
5373 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5374#endif
5375
5376#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5377 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5378 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5379 pVCpu->hm.s.vmx.HCPhysVmcs));
5380 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5381 pCache->TestOut.HCPhysVmcs));
5382 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5383 pCache->TestOut.pCache));
5384 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5385 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5386 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5387 pCache->TestOut.pCtx));
5388 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5389#endif
5390 NOREF(pCtx);
5391 return rc;
5392}
5393
5394
5395/**
5396 * Initialize the VMCS-Read cache.
5397 *
5398 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5399 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5400 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5401 * (those that have a 32-bit FULL & HIGH part).
5402 *
5403 * @returns VBox status code.
5404 * @param pVCpu The cross context virtual CPU structure.
5405 */
5406static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5407{
5408#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5409 do { \
5410 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5411 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5412 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5413 ++cReadFields; \
5414 } while (0)
5415
5416 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5417 uint32_t cReadFields = 0;
5418
5419 /*
5420 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5421 * and serve to indicate exceptions to the rules.
5422 */
5423
5424 /* Guest-natural selector base fields. */
5425#if 0
5426 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5429#endif
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5442#if 0
5443 /* Unused natural width guest-state fields. */
5444 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5446#endif
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5449
5450 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5451 these 64-bit fields (using "FULL" and "HIGH" fields). */
5452#if 0
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5462#endif
5463
5464 /* Natural width guest-state fields. */
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5466#if 0
5467 /* Currently unused field. */
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5469#endif
5470
5471 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5472 {
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5474 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5475 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5476 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5477 }
5478 else
5479 {
5480 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5481 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5482 }
5483
5484#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5485 return VINF_SUCCESS;
5486}
5487
5488
5489/**
5490 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5491 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5492 * darwin, running 64-bit guests).
5493 *
5494 * @returns VBox status code.
5495 * @param pVCpu The cross context virtual CPU structure.
5496 * @param idxField The VMCS field encoding.
5497 * @param u64Val 16, 32 or 64-bit value.
5498 */
5499VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5500{
5501 int rc;
5502 switch (idxField)
5503 {
5504 /*
5505 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5506 */
5507 /* 64-bit Control fields. */
5508 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5509 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5510 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5511 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5512 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5513 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5514 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5515 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5516 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5517 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5518 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5519 case VMX_VMCS64_CTRL_EPTP_FULL:
5520 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5521 /* 64-bit Guest-state fields. */
5522 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5523 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5524 case VMX_VMCS64_GUEST_PAT_FULL:
5525 case VMX_VMCS64_GUEST_EFER_FULL:
5526 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5527 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5528 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5529 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5530 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5531 /* 64-bit Host-state fields. */
5532 case VMX_VMCS64_HOST_PAT_FULL:
5533 case VMX_VMCS64_HOST_EFER_FULL:
5534 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5535 {
5536 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5537 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5538 break;
5539 }
5540
5541 /*
5542 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5543 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5544 */
5545 /* Natural-width Guest-state fields. */
5546 case VMX_VMCS_GUEST_CR3:
5547 case VMX_VMCS_GUEST_ES_BASE:
5548 case VMX_VMCS_GUEST_CS_BASE:
5549 case VMX_VMCS_GUEST_SS_BASE:
5550 case VMX_VMCS_GUEST_DS_BASE:
5551 case VMX_VMCS_GUEST_FS_BASE:
5552 case VMX_VMCS_GUEST_GS_BASE:
5553 case VMX_VMCS_GUEST_LDTR_BASE:
5554 case VMX_VMCS_GUEST_TR_BASE:
5555 case VMX_VMCS_GUEST_GDTR_BASE:
5556 case VMX_VMCS_GUEST_IDTR_BASE:
5557 case VMX_VMCS_GUEST_RSP:
5558 case VMX_VMCS_GUEST_RIP:
5559 case VMX_VMCS_GUEST_SYSENTER_ESP:
5560 case VMX_VMCS_GUEST_SYSENTER_EIP:
5561 {
5562 if (!(RT_HI_U32(u64Val)))
5563 {
5564 /* If this field is 64-bit, VT-x will zero out the top bits. */
5565 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5566 }
5567 else
5568 {
5569 /* Assert that only the 32->64 switcher case should ever come here. */
5570 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5571 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5572 }
5573 break;
5574 }
5575
5576 default:
5577 {
5578 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5579 rc = VERR_INVALID_PARAMETER;
5580 break;
5581 }
5582 }
5583 AssertRCReturn(rc, rc);
5584 return rc;
5585}
5586
5587
5588/**
5589 * Queue up a VMWRITE by using the VMCS write cache.
5590 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5591 *
5592 * @param pVCpu The cross context virtual CPU structure.
5593 * @param idxField The VMCS field encoding.
5594 * @param u64Val 16, 32 or 64-bit value.
5595 */
5596VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5597{
5598 AssertPtr(pVCpu);
5599 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5600
5601 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5602 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5603
5604 /* Make sure there are no duplicates. */
5605 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5606 {
5607 if (pCache->Write.aField[i] == idxField)
5608 {
5609 pCache->Write.aFieldVal[i] = u64Val;
5610 return VINF_SUCCESS;
5611 }
5612 }
5613
5614 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5615 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5616 pCache->Write.cValidEntries++;
5617 return VINF_SUCCESS;
5618}
5619#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5620
5621
5622/**
5623 * Sets up the usage of TSC-offsetting and updates the VMCS.
5624 *
5625 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5626 * VMX preemption timer.
5627 *
5628 * @returns VBox status code.
5629 * @param pVCpu The cross context virtual CPU structure.
5630 *
5631 * @remarks No-long-jump zone!!!
5632 */
5633static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5634{
5635 bool fOffsettedTsc;
5636 bool fParavirtTsc;
5637 PVM pVM = pVCpu->CTX_SUFF(pVM);
5638 uint64_t uTscOffset;
5639 if (pVM->hm.s.vmx.fUsePreemptTimer)
5640 {
5641 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5642
5643 /* Make sure the returned values have sane upper and lower boundaries. */
5644 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5645 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5646 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5647 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5648
5649 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5650 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5651 AssertRC(rc);
5652 }
5653 else
5654 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5655
5656 /** @todo later optimize this to be done elsewhere and not before every
5657 * VM-entry. */
5658 if (fParavirtTsc)
5659 {
5660 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5661 information before every VM-entry, hence disable it for performance sake. */
5662#if 0
5663 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5664 AssertRC(rc);
5665#endif
5666 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5667 }
5668
5669 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5670 if ( fOffsettedTsc
5671 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5672 {
5673 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5674 {
5675 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5676 AssertRC(rc);
5677 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5678 }
5679
5680 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5681 {
5682 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5683 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5684 AssertRC(rc);
5685 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5686 }
5687 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5688 }
5689 else
5690 {
5691 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5692 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5693 {
5694 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5695 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5696 AssertRC(rc);
5697 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5698 }
5699 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5700 }
5701}
5702
5703
5704/**
5705 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5706 * VM-exit interruption info type.
5707 *
5708 * @returns The IEM exception flags.
5709 * @param uVector The event vector.
5710 * @param uVmxVectorType The VMX event type.
5711 *
5712 * @remarks This function currently only constructs flags required for
5713 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5714 * and CR2 aspects of an exception are not included).
5715 */
5716static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5717{
5718 uint32_t fIemXcptFlags;
5719 switch (uVmxVectorType)
5720 {
5721 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5722 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5724 break;
5725
5726 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5727 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5728 break;
5729
5730 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5731 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5732 break;
5733
5734 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5735 {
5736 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5737 if (uVector == X86_XCPT_BP)
5738 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5739 else if (uVector == X86_XCPT_OF)
5740 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5741 else
5742 {
5743 fIemXcptFlags = 0;
5744 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5745 }
5746 break;
5747 }
5748
5749 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5750 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5751 break;
5752
5753 default:
5754 fIemXcptFlags = 0;
5755 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5756 break;
5757 }
5758 return fIemXcptFlags;
5759}
5760
5761
5762/**
5763 * Sets an event as a pending event to be injected into the guest.
5764 *
5765 * @param pVCpu The cross context virtual CPU structure.
5766 * @param u32IntInfo The VM-entry interruption-information field.
5767 * @param cbInstr The VM-entry instruction length in bytes (for software
5768 * interrupts, exceptions and privileged software
5769 * exceptions).
5770 * @param u32ErrCode The VM-entry exception error code.
5771 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5772 * page-fault.
5773 *
5774 * @remarks Statistics counter assumes this is a guest event being injected or
5775 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5776 * always incremented.
5777 */
5778DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5779 RTGCUINTPTR GCPtrFaultAddress)
5780{
5781 Assert(!pVCpu->hm.s.Event.fPending);
5782 pVCpu->hm.s.Event.fPending = true;
5783 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5784 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5785 pVCpu->hm.s.Event.cbInstr = cbInstr;
5786 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5787}
5788
5789
5790/**
5791 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5792 *
5793 * @param pVCpu The cross context virtual CPU structure.
5794 */
5795DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5796{
5797 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5798 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5799 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5800 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5801}
5802
5803
5804/**
5805 * Handle a condition that occurred while delivering an event through the guest
5806 * IDT.
5807 *
5808 * @returns Strict VBox status code (i.e. informational status codes too).
5809 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5810 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5811 * to continue execution of the guest which will delivery the \#DF.
5812 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5813 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5814 *
5815 * @param pVCpu The cross context virtual CPU structure.
5816 * @param pVmxTransient Pointer to the VMX transient structure.
5817 *
5818 * @remarks No-long-jump zone!!!
5819 */
5820static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5821{
5822 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5823
5824 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5825 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5826 AssertRCReturn(rc2, rc2);
5827
5828 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5829 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5830 {
5831 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5832 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5833
5834 /*
5835 * If the event was a software interrupt (generated with INT n) or a software exception
5836 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5837 * can handle the VM-exit and continue guest execution which will re-execute the
5838 * instruction rather than re-injecting the exception, as that can cause premature
5839 * trips to ring-3 before injection and involve TRPM which currently has no way of
5840 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5841 * the problem).
5842 */
5843 IEMXCPTRAISE enmRaise;
5844 IEMXCPTRAISEINFO fRaiseInfo;
5845 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5846 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5847 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5848 {
5849 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5850 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5851 }
5852 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5853 {
5854 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5855 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5856 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5857 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5858 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5859 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5860 uExitVectorType), VERR_VMX_IPE_5);
5861
5862 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5863
5864 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5865 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5866 {
5867 pVmxTransient->fVectoringPF = true;
5868 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5869 }
5870 }
5871 else
5872 {
5873 /*
5874 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5875 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5876 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5877 */
5878 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5879 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5880 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5881 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5882 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5883 }
5884
5885 /*
5886 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5887 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5888 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5889 * subsequent VM-entry would fail.
5890 *
5891 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5892 */
5893 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5894 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5895 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5896 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5897 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5898 {
5899 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5900 }
5901
5902 switch (enmRaise)
5903 {
5904 case IEMXCPTRAISE_CURRENT_XCPT:
5905 {
5906 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5907 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5908 Assert(rcStrict == VINF_SUCCESS);
5909 break;
5910 }
5911
5912 case IEMXCPTRAISE_PREV_EVENT:
5913 {
5914 uint32_t u32ErrCode;
5915 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5916 {
5917 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5918 AssertRCReturn(rc2, rc2);
5919 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5920 }
5921 else
5922 u32ErrCode = 0;
5923
5924 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5925 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5926 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5927 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5928
5929 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5930 pVCpu->hm.s.Event.u32ErrCode));
5931 Assert(rcStrict == VINF_SUCCESS);
5932 break;
5933 }
5934
5935 case IEMXCPTRAISE_REEXEC_INSTR:
5936 Assert(rcStrict == VINF_SUCCESS);
5937 break;
5938
5939 case IEMXCPTRAISE_DOUBLE_FAULT:
5940 {
5941 /*
5942 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5943 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5944 */
5945 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5946 {
5947 pVmxTransient->fVectoringDoublePF = true;
5948 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5949 pVCpu->cpum.GstCtx.cr2));
5950 rcStrict = VINF_SUCCESS;
5951 }
5952 else
5953 {
5954 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5955 hmR0VmxSetPendingXcptDF(pVCpu);
5956 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5957 uIdtVector, uExitVector));
5958 rcStrict = VINF_HM_DOUBLE_FAULT;
5959 }
5960 break;
5961 }
5962
5963 case IEMXCPTRAISE_TRIPLE_FAULT:
5964 {
5965 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5966 rcStrict = VINF_EM_RESET;
5967 break;
5968 }
5969
5970 case IEMXCPTRAISE_CPU_HANG:
5971 {
5972 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5973 rcStrict = VERR_EM_GUEST_CPU_HANG;
5974 break;
5975 }
5976
5977 default:
5978 {
5979 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5980 rcStrict = VERR_VMX_IPE_2;
5981 break;
5982 }
5983 }
5984 }
5985 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5986 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5987 && uExitVector != X86_XCPT_DF
5988 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5989 {
5990 /*
5991 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5992 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5993 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5994 */
5995 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5996 {
5997 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5998 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5999 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6000 }
6001 }
6002
6003 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6004 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6005 return rcStrict;
6006}
6007
6008
6009/**
6010 * Imports a guest segment register from the current VMCS into
6011 * the guest-CPU context.
6012 *
6013 * @returns VBox status code.
6014 * @param pVCpu The cross context virtual CPU structure.
6015 * @param idxSel Index of the selector in the VMCS.
6016 * @param idxLimit Index of the segment limit in the VMCS.
6017 * @param idxBase Index of the segment base in the VMCS.
6018 * @param idxAccess Index of the access rights of the segment in the VMCS.
6019 * @param pSelReg Pointer to the segment selector.
6020 *
6021 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6022 * do not log!
6023 *
6024 * @remarks Never call this function directly!!! Use the
6025 * HMVMX_IMPORT_SREG() macro as that takes care
6026 * of whether to read from the VMCS cache or not.
6027 */
6028static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6029 PCPUMSELREG pSelReg)
6030{
6031 NOREF(pVCpu);
6032
6033 uint32_t u32Sel;
6034 uint32_t u32Limit;
6035 uint32_t u32Attr;
6036 uint64_t u64Base;
6037 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6038 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6039 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6040 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6041 AssertRCReturn(rc, rc);
6042
6043 pSelReg->Sel = (uint16_t)u32Sel;
6044 pSelReg->ValidSel = (uint16_t)u32Sel;
6045 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6046 pSelReg->u32Limit = u32Limit;
6047 pSelReg->u64Base = u64Base;
6048 pSelReg->Attr.u = u32Attr;
6049
6050 /*
6051 * If VT-x marks the segment as unusable, most other bits remain undefined:
6052 * - For CS the L, D and G bits have meaning.
6053 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6054 * - For the remaining data segments no bits are defined.
6055 *
6056 * The present bit and the unusable bit has been observed to be set at the
6057 * same time (the selector was supposed to be invalid as we started executing
6058 * a V8086 interrupt in ring-0).
6059 *
6060 * What should be important for the rest of the VBox code, is that the P bit is
6061 * cleared. Some of the other VBox code recognizes the unusable bit, but
6062 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6063 * safe side here, we'll strip off P and other bits we don't care about. If
6064 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6065 *
6066 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6067 */
6068 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6069 {
6070 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6071
6072 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6073 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6074 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6075#ifdef VBOX_STRICT
6076 VMMRZCallRing3Disable(pVCpu);
6077 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6078# ifdef DEBUG_bird
6079 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6080 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6081 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6082# endif
6083 VMMRZCallRing3Enable(pVCpu);
6084#endif
6085 }
6086 return VINF_SUCCESS;
6087}
6088
6089
6090/**
6091 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6092 *
6093 * @returns VBox status code.
6094 * @param pVCpu The cross context virtual CPU structure.
6095 *
6096 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6097 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6098 * instead!!!
6099 */
6100DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6101{
6102 uint64_t u64Val;
6103 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6104 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6105 {
6106 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6107 if (RT_SUCCESS(rc))
6108 {
6109 pCtx->rip = u64Val;
6110 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6111 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6112 }
6113 return rc;
6114 }
6115 return VINF_SUCCESS;
6116}
6117
6118
6119/**
6120 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6121 *
6122 * @returns VBox status code.
6123 * @param pVCpu The cross context virtual CPU structure.
6124 *
6125 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6126 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6127 * instead!!!
6128 */
6129DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6130{
6131 uint32_t u32Val;
6132 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6133 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6134 {
6135 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6136 if (RT_SUCCESS(rc))
6137 {
6138 pCtx->eflags.u32 = u32Val;
6139
6140 /* Restore eflags for real-on-v86-mode hack. */
6141 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6142 {
6143 pCtx->eflags.Bits.u1VM = 0;
6144 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6145 }
6146 }
6147 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6148 return rc;
6149 }
6150 return VINF_SUCCESS;
6151}
6152
6153
6154/**
6155 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6156 * context.
6157 *
6158 * @returns VBox status code.
6159 * @param pVCpu The cross context virtual CPU structure.
6160 *
6161 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6162 * do not log!
6163 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6164 * instead!!!
6165 */
6166DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6167{
6168 uint32_t u32Val;
6169 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6170 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6171 if (RT_SUCCESS(rc))
6172 {
6173 /*
6174 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6175 * might need them in hmR0VmxEvaluatePendingEvent().
6176 */
6177 if (!u32Val)
6178 {
6179 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6180 {
6181 rc = hmR0VmxImportGuestRip(pVCpu);
6182 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6183 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6184 }
6185
6186 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6187 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6188 }
6189 else
6190 {
6191 rc = hmR0VmxImportGuestRip(pVCpu);
6192 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6193
6194 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6195 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6196 {
6197 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6198 }
6199 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6200 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6201
6202 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6203 {
6204 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6205 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6206 }
6207 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6208 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6209 }
6210 }
6211 return rc;
6212}
6213
6214
6215/**
6216 * Worker for VMXR0ImportStateOnDemand.
6217 *
6218 * @returns VBox status code.
6219 * @param pVCpu The cross context virtual CPU structure.
6220 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6221 */
6222static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6223{
6224#define VMXLOCAL_BREAK_RC(a_rc) \
6225 if (RT_FAILURE(a_rc)) \
6226 break
6227
6228 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6229
6230 int rc = VINF_SUCCESS;
6231 PVM pVM = pVCpu->CTX_SUFF(pVM);
6232 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6233 uint64_t u64Val;
6234 uint32_t u32Val;
6235
6236 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6237
6238 /*
6239 * We disable interrupts to make the updating of the state and in particular
6240 * the fExtrn modification atomic wrt to preemption hooks.
6241 */
6242 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6243
6244 fWhat &= pCtx->fExtrn;
6245 if (fWhat)
6246 {
6247 do
6248 {
6249 if (fWhat & CPUMCTX_EXTRN_RIP)
6250 {
6251 rc = hmR0VmxImportGuestRip(pVCpu);
6252 VMXLOCAL_BREAK_RC(rc);
6253 }
6254
6255 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6256 {
6257 rc = hmR0VmxImportGuestRFlags(pVCpu);
6258 VMXLOCAL_BREAK_RC(rc);
6259 }
6260
6261 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6262 {
6263 rc = hmR0VmxImportGuestIntrState(pVCpu);
6264 VMXLOCAL_BREAK_RC(rc);
6265 }
6266
6267 if (fWhat & CPUMCTX_EXTRN_RSP)
6268 {
6269 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6270 VMXLOCAL_BREAK_RC(rc);
6271 pCtx->rsp = u64Val;
6272 }
6273
6274 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6275 {
6276 if (fWhat & CPUMCTX_EXTRN_CS)
6277 {
6278 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6279 rc |= hmR0VmxImportGuestRip(pVCpu);
6280 VMXLOCAL_BREAK_RC(rc);
6281 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6282 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6283 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6284 }
6285 if (fWhat & CPUMCTX_EXTRN_SS)
6286 {
6287 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6288 VMXLOCAL_BREAK_RC(rc);
6289 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6290 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6291 }
6292 if (fWhat & CPUMCTX_EXTRN_DS)
6293 {
6294 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6295 VMXLOCAL_BREAK_RC(rc);
6296 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6297 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6298 }
6299 if (fWhat & CPUMCTX_EXTRN_ES)
6300 {
6301 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6302 VMXLOCAL_BREAK_RC(rc);
6303 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6304 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6305 }
6306 if (fWhat & CPUMCTX_EXTRN_FS)
6307 {
6308 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6309 VMXLOCAL_BREAK_RC(rc);
6310 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6311 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6312 }
6313 if (fWhat & CPUMCTX_EXTRN_GS)
6314 {
6315 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6316 VMXLOCAL_BREAK_RC(rc);
6317 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6318 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6319 }
6320 }
6321
6322 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6323 {
6324 if (fWhat & CPUMCTX_EXTRN_LDTR)
6325 {
6326 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6327 VMXLOCAL_BREAK_RC(rc);
6328 }
6329
6330 if (fWhat & CPUMCTX_EXTRN_GDTR)
6331 {
6332 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6333 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6334 VMXLOCAL_BREAK_RC(rc);
6335 pCtx->gdtr.pGdt = u64Val;
6336 pCtx->gdtr.cbGdt = u32Val;
6337 }
6338
6339 /* Guest IDTR. */
6340 if (fWhat & CPUMCTX_EXTRN_IDTR)
6341 {
6342 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6343 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6344 VMXLOCAL_BREAK_RC(rc);
6345 pCtx->idtr.pIdt = u64Val;
6346 pCtx->idtr.cbIdt = u32Val;
6347 }
6348
6349 /* Guest TR. */
6350 if (fWhat & CPUMCTX_EXTRN_TR)
6351 {
6352 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6353 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6354 {
6355 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6356 VMXLOCAL_BREAK_RC(rc);
6357 }
6358 }
6359 }
6360
6361 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6362 {
6363 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6364 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6365 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6366 pCtx->SysEnter.cs = u32Val;
6367 VMXLOCAL_BREAK_RC(rc);
6368 }
6369
6370#if HC_ARCH_BITS == 64
6371 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6372 {
6373 if ( pVM->hm.s.fAllow64BitGuests
6374 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6375 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6376 }
6377
6378 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6379 {
6380 if ( pVM->hm.s.fAllow64BitGuests
6381 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6382 {
6383 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6384 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6385 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6386 }
6387 }
6388#endif
6389
6390 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6391#if HC_ARCH_BITS == 32
6392 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6393#endif
6394 )
6395 {
6396 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6397 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6398 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6399 {
6400 switch (pMsr->u32Msr)
6401 {
6402#if HC_ARCH_BITS == 32
6403 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6404 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6405 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6406 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6407#endif
6408 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6409 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6410 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6411 default:
6412 {
6413 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6414 ASMSetFlags(fEFlags);
6415 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6416 cMsrs));
6417 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6418 }
6419 }
6420 }
6421 }
6422
6423 if (fWhat & CPUMCTX_EXTRN_DR7)
6424 {
6425 if (!pVCpu->hm.s.fUsingHyperDR7)
6426 {
6427 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6428 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6429 VMXLOCAL_BREAK_RC(rc);
6430 pCtx->dr[7] = u32Val;
6431 }
6432 }
6433
6434 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6435 {
6436 uint32_t u32Shadow;
6437 if (fWhat & CPUMCTX_EXTRN_CR0)
6438 {
6439 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6440 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6441 VMXLOCAL_BREAK_RC(rc);
6442 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6443 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6444 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6445 CPUMSetGuestCR0(pVCpu, u32Val);
6446 VMMRZCallRing3Enable(pVCpu);
6447 }
6448
6449 if (fWhat & CPUMCTX_EXTRN_CR4)
6450 {
6451 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6452 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6453 VMXLOCAL_BREAK_RC(rc);
6454 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6455 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6456 CPUMSetGuestCR4(pVCpu, u32Val);
6457 }
6458
6459 if (fWhat & CPUMCTX_EXTRN_CR3)
6460 {
6461 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6462 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6463 || ( pVM->hm.s.fNestedPaging
6464 && CPUMIsGuestPagingEnabledEx(pCtx)))
6465 {
6466 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6467 if (pCtx->cr3 != u64Val)
6468 {
6469 CPUMSetGuestCR3(pVCpu, u64Val);
6470 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6471 }
6472
6473 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6474 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6475 if (CPUMIsGuestInPAEModeEx(pCtx))
6476 {
6477 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6478 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6479 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6480 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6481 VMXLOCAL_BREAK_RC(rc);
6482 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6483 }
6484 }
6485 }
6486 }
6487 } while (0);
6488
6489 if (RT_SUCCESS(rc))
6490 {
6491 /* Update fExtrn. */
6492 pCtx->fExtrn &= ~fWhat;
6493
6494 /* If everything has been imported, clear the HM keeper bit. */
6495 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6496 {
6497 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6498 Assert(!pCtx->fExtrn);
6499 }
6500 }
6501 }
6502 else
6503 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6504
6505 ASMSetFlags(fEFlags);
6506
6507 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6508
6509 /*
6510 * Honor any pending CR3 updates.
6511 *
6512 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6513 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6514 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6515 *
6516 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6517 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6518 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6519 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6520 *
6521 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6522 */
6523 if (VMMRZCallRing3IsEnabled(pVCpu))
6524 {
6525 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6526 {
6527 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6528 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6529 }
6530
6531 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6532 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6533
6534 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6535 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6536 }
6537
6538 return VINF_SUCCESS;
6539#undef VMXLOCAL_BREAK_RC
6540}
6541
6542
6543/**
6544 * Saves the guest state from the VMCS into the guest-CPU context.
6545 *
6546 * @returns VBox status code.
6547 * @param pVCpu The cross context virtual CPU structure.
6548 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6549 */
6550VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6551{
6552 return hmR0VmxImportGuestState(pVCpu, fWhat);
6553}
6554
6555
6556/**
6557 * Check per-VM and per-VCPU force flag actions that require us to go back to
6558 * ring-3 for one reason or another.
6559 *
6560 * @returns Strict VBox status code (i.e. informational status codes too)
6561 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6562 * ring-3.
6563 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6564 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6565 * interrupts)
6566 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6567 * all EMTs to be in ring-3.
6568 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6569 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6570 * to the EM loop.
6571 *
6572 * @param pVCpu The cross context virtual CPU structure.
6573 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6574 */
6575static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6576{
6577 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6578
6579 /*
6580 * Anything pending? Should be more likely than not if we're doing a good job.
6581 */
6582 PVM pVM = pVCpu->CTX_SUFF(pVM);
6583 if ( !fStepping
6584 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6585 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6586 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6587 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6588 return VINF_SUCCESS;
6589
6590 /* Pending PGM C3 sync. */
6591 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6592 {
6593 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6594 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6595 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6596 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6597 if (rcStrict2 != VINF_SUCCESS)
6598 {
6599 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6600 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6601 return rcStrict2;
6602 }
6603 }
6604
6605 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6606 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6607 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6608 {
6609 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6610 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6611 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6612 return rc2;
6613 }
6614
6615 /* Pending VM request packets, such as hardware interrupts. */
6616 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6617 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6618 {
6619 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6620 return VINF_EM_PENDING_REQUEST;
6621 }
6622
6623 /* Pending PGM pool flushes. */
6624 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6625 {
6626 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6627 return VINF_PGM_POOL_FLUSH_PENDING;
6628 }
6629
6630 /* Pending DMA requests. */
6631 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6632 {
6633 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6634 return VINF_EM_RAW_TO_R3;
6635 }
6636
6637 return VINF_SUCCESS;
6638}
6639
6640
6641/**
6642 * Converts any TRPM trap into a pending HM event. This is typically used when
6643 * entering from ring-3 (not longjmp returns).
6644 *
6645 * @param pVCpu The cross context virtual CPU structure.
6646 */
6647static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6648{
6649 Assert(TRPMHasTrap(pVCpu));
6650 Assert(!pVCpu->hm.s.Event.fPending);
6651
6652 uint8_t uVector;
6653 TRPMEVENT enmTrpmEvent;
6654 RTGCUINT uErrCode;
6655 RTGCUINTPTR GCPtrFaultAddress;
6656 uint8_t cbInstr;
6657
6658 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6659 AssertRC(rc);
6660
6661 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6662 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6663 if (enmTrpmEvent == TRPM_TRAP)
6664 {
6665 switch (uVector)
6666 {
6667 case X86_XCPT_NMI:
6668 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6669 break;
6670
6671 case X86_XCPT_BP:
6672 case X86_XCPT_OF:
6673 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6674 break;
6675
6676 case X86_XCPT_PF:
6677 case X86_XCPT_DF:
6678 case X86_XCPT_TS:
6679 case X86_XCPT_NP:
6680 case X86_XCPT_SS:
6681 case X86_XCPT_GP:
6682 case X86_XCPT_AC:
6683 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6684 RT_FALL_THRU();
6685 default:
6686 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6687 break;
6688 }
6689 }
6690 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6691 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6692 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6693 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6694 else
6695 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6696
6697 rc = TRPMResetTrap(pVCpu);
6698 AssertRC(rc);
6699 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6700 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6701
6702 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6703}
6704
6705
6706/**
6707 * Converts the pending HM event into a TRPM trap.
6708 *
6709 * @param pVCpu The cross context virtual CPU structure.
6710 */
6711static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6712{
6713 Assert(pVCpu->hm.s.Event.fPending);
6714
6715 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6716 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6717 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6718 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6719
6720 /* If a trap was already pending, we did something wrong! */
6721 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6722
6723 TRPMEVENT enmTrapType;
6724 switch (uVectorType)
6725 {
6726 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6727 enmTrapType = TRPM_HARDWARE_INT;
6728 break;
6729
6730 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6731 enmTrapType = TRPM_SOFTWARE_INT;
6732 break;
6733
6734 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6735 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6736 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6737 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6738 enmTrapType = TRPM_TRAP;
6739 break;
6740
6741 default:
6742 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6743 enmTrapType = TRPM_32BIT_HACK;
6744 break;
6745 }
6746
6747 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6748
6749 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6750 AssertRC(rc);
6751
6752 if (fErrorCodeValid)
6753 TRPMSetErrorCode(pVCpu, uErrorCode);
6754
6755 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6756 && uVector == X86_XCPT_PF)
6757 {
6758 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6759 }
6760 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6761 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6762 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6763 {
6764 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6765 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6766 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6767 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6768 }
6769
6770 /* Clear any pending events from the VMCS. */
6771 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6772 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6773
6774 /* We're now done converting the pending event. */
6775 pVCpu->hm.s.Event.fPending = false;
6776}
6777
6778
6779/**
6780 * Does the necessary state syncing before returning to ring-3 for any reason
6781 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6782 *
6783 * @returns VBox status code.
6784 * @param pVCpu The cross context virtual CPU structure.
6785 * @param fImportState Whether to import the guest state from the VMCS back
6786 * to the guest-CPU context.
6787 *
6788 * @remarks No-long-jmp zone!!!
6789 */
6790static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6791{
6792 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6793 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6794
6795 RTCPUID idCpu = RTMpCpuId();
6796 Log4Func(("HostCpuId=%u\n", idCpu));
6797
6798 /*
6799 * !!! IMPORTANT !!!
6800 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6801 */
6802
6803 /* Save the guest state if necessary. */
6804 if (fImportState)
6805 {
6806 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6807 AssertRCReturn(rc, rc);
6808 }
6809
6810 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6811 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6812 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6813
6814 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6815#ifdef VBOX_STRICT
6816 if (CPUMIsHyperDebugStateActive(pVCpu))
6817 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6818#endif
6819 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6820 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6821 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6822
6823#if HC_ARCH_BITS == 64
6824 /* Restore host-state bits that VT-x only restores partially. */
6825 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6826 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6827 {
6828 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6829 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6830 }
6831 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6832#endif
6833
6834 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6835 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6836 {
6837 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6838 if (!fImportState)
6839 {
6840 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6841 AssertRCReturn(rc, rc);
6842 }
6843 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6844 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6845 }
6846 else
6847 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6848
6849 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6850 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6851
6852 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6853 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6854 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6855 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6856 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6857 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6858 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6859 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6860 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6861
6862 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6863
6864 /** @todo This partially defeats the purpose of having preemption hooks.
6865 * The problem is, deregistering the hooks should be moved to a place that
6866 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6867 * context.
6868 */
6869 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6870 {
6871 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6872 AssertRCReturn(rc, rc);
6873
6874 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6875 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6876 }
6877 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6878 NOREF(idCpu);
6879
6880 return VINF_SUCCESS;
6881}
6882
6883
6884/**
6885 * Leaves the VT-x session.
6886 *
6887 * @returns VBox status code.
6888 * @param pVCpu The cross context virtual CPU structure.
6889 *
6890 * @remarks No-long-jmp zone!!!
6891 */
6892static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6893{
6894 HM_DISABLE_PREEMPT(pVCpu);
6895 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6896 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6897 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6898
6899 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6900 and done this from the VMXR0ThreadCtxCallback(). */
6901 if (!pVCpu->hm.s.fLeaveDone)
6902 {
6903 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6904 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6905 pVCpu->hm.s.fLeaveDone = true;
6906 }
6907 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6908
6909 /*
6910 * !!! IMPORTANT !!!
6911 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6912 */
6913
6914 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6915 /** @todo Deregistering here means we need to VMCLEAR always
6916 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6917 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6918 VMMR0ThreadCtxHookDisable(pVCpu);
6919
6920 /* Leave HM context. This takes care of local init (term). */
6921 int rc = HMR0LeaveCpu(pVCpu);
6922
6923 HM_RESTORE_PREEMPT();
6924 return rc;
6925}
6926
6927
6928/**
6929 * Does the necessary state syncing before doing a longjmp to ring-3.
6930 *
6931 * @returns VBox status code.
6932 * @param pVCpu The cross context virtual CPU structure.
6933 *
6934 * @remarks No-long-jmp zone!!!
6935 */
6936DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6937{
6938 return hmR0VmxLeaveSession(pVCpu);
6939}
6940
6941
6942/**
6943 * Take necessary actions before going back to ring-3.
6944 *
6945 * An action requires us to go back to ring-3. This function does the necessary
6946 * steps before we can safely return to ring-3. This is not the same as longjmps
6947 * to ring-3, this is voluntary and prepares the guest so it may continue
6948 * executing outside HM (recompiler/IEM).
6949 *
6950 * @returns VBox status code.
6951 * @param pVCpu The cross context virtual CPU structure.
6952 * @param rcExit The reason for exiting to ring-3. Can be
6953 * VINF_VMM_UNKNOWN_RING3_CALL.
6954 */
6955static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6956{
6957 Assert(pVCpu);
6958 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6959
6960 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6961 {
6962 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6963 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6964 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6965 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6966 }
6967
6968 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6969 VMMRZCallRing3Disable(pVCpu);
6970 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6971
6972 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6973 if (pVCpu->hm.s.Event.fPending)
6974 {
6975 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6976 Assert(!pVCpu->hm.s.Event.fPending);
6977 }
6978
6979 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6980 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6981
6982 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6983 and if we're injecting an event we should have a TRPM trap pending. */
6984 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6985#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6986 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6987#endif
6988
6989 /* Save guest state and restore host state bits. */
6990 int rc = hmR0VmxLeaveSession(pVCpu);
6991 AssertRCReturn(rc, rc);
6992 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6993 /* Thread-context hooks are unregistered at this point!!! */
6994
6995 /* Sync recompiler state. */
6996 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6997 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6998 | CPUM_CHANGED_LDTR
6999 | CPUM_CHANGED_GDTR
7000 | CPUM_CHANGED_IDTR
7001 | CPUM_CHANGED_TR
7002 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7003 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7004 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7005 {
7006 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7007 }
7008
7009 Assert(!pVCpu->hm.s.fClearTrapFlag);
7010
7011 /* Update the exit-to-ring 3 reason. */
7012 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7013
7014 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7015 if (rcExit != VINF_EM_RAW_INTERRUPT)
7016 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7017
7018 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7019
7020 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7021 VMMRZCallRing3RemoveNotification(pVCpu);
7022 VMMRZCallRing3Enable(pVCpu);
7023
7024 return rc;
7025}
7026
7027
7028/**
7029 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7030 * longjump to ring-3 and possibly get preempted.
7031 *
7032 * @returns VBox status code.
7033 * @param pVCpu The cross context virtual CPU structure.
7034 * @param enmOperation The operation causing the ring-3 longjump.
7035 * @param pvUser User argument, currently unused, NULL.
7036 */
7037static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7038{
7039 RT_NOREF(pvUser);
7040 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7041 {
7042 /*
7043 * !!! IMPORTANT !!!
7044 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7045 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7046 */
7047 VMMRZCallRing3RemoveNotification(pVCpu);
7048 VMMRZCallRing3Disable(pVCpu);
7049 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7050 RTThreadPreemptDisable(&PreemptState);
7051
7052 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7053 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7054 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7055
7056#if HC_ARCH_BITS == 64
7057 /* Restore host-state bits that VT-x only restores partially. */
7058 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7059 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7060 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7061 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7062#endif
7063
7064 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7065 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7066 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7067
7068 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7069 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7070 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7071 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7072 {
7073 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7074 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7075 }
7076
7077 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7078 VMMR0ThreadCtxHookDisable(pVCpu);
7079 HMR0LeaveCpu(pVCpu);
7080 RTThreadPreemptRestore(&PreemptState);
7081 return VINF_SUCCESS;
7082 }
7083
7084 Assert(pVCpu);
7085 Assert(pvUser);
7086 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7087 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7088
7089 VMMRZCallRing3Disable(pVCpu);
7090 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7091
7092 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7093
7094 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7095 AssertRCReturn(rc, rc);
7096
7097 VMMRZCallRing3Enable(pVCpu);
7098 return VINF_SUCCESS;
7099}
7100
7101
7102/**
7103 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7104 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7105 *
7106 * @param pVCpu The cross context virtual CPU structure.
7107 */
7108DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7109{
7110 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7111 {
7112 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7113 {
7114 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7115 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7116 AssertRC(rc);
7117 Log4Func(("Setup interrupt-window exiting\n"));
7118 }
7119 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7120}
7121
7122
7123/**
7124 * Clears the interrupt-window exiting control in the VMCS.
7125 *
7126 * @param pVCpu The cross context virtual CPU structure.
7127 */
7128DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7129{
7130 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7131 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7132 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7133 AssertRC(rc);
7134 Log4Func(("Cleared interrupt-window exiting\n"));
7135}
7136
7137
7138/**
7139 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7140 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7141 *
7142 * @param pVCpu The cross context virtual CPU structure.
7143 */
7144DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7145{
7146 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7147 {
7148 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7149 {
7150 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7151 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7152 AssertRC(rc);
7153 Log4Func(("Setup NMI-window exiting\n"));
7154 }
7155 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7156}
7157
7158
7159/**
7160 * Clears the NMI-window exiting control in the VMCS.
7161 *
7162 * @param pVCpu The cross context virtual CPU structure.
7163 */
7164DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7165{
7166 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7167 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7168 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7169 AssertRC(rc);
7170 Log4Func(("Cleared NMI-window exiting\n"));
7171}
7172
7173
7174/**
7175 * Evaluates the event to be delivered to the guest and sets it as the pending
7176 * event.
7177 *
7178 * @returns The VT-x guest-interruptibility state.
7179 * @param pVCpu The cross context virtual CPU structure.
7180 */
7181static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7182{
7183 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7184 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7185 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7186 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7187 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7188 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7189
7190 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7191 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7192 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7193 Assert(!TRPMHasTrap(pVCpu));
7194
7195 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7196 APICUpdatePendingInterrupts(pVCpu);
7197
7198 /*
7199 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7200 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7201 */
7202 /** @todo SMI. SMIs take priority over NMIs. */
7203 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7204 {
7205 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7206 if ( !pVCpu->hm.s.Event.fPending
7207 && !fBlockNmi
7208 && !fBlockSti
7209 && !fBlockMovSS)
7210 {
7211 Log4Func(("Pending NMI\n"));
7212 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7213 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7214
7215 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7216 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7217 }
7218 else
7219 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7220 }
7221 /*
7222 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7223 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7224 */
7225 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7226 && !pVCpu->hm.s.fSingleInstruction)
7227 {
7228 Assert(!DBGFIsStepping(pVCpu));
7229 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7230 AssertRCReturn(rc, 0);
7231 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7232 if ( !pVCpu->hm.s.Event.fPending
7233 && !fBlockInt
7234 && !fBlockSti
7235 && !fBlockMovSS)
7236 {
7237 uint8_t u8Interrupt;
7238 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7239 if (RT_SUCCESS(rc))
7240 {
7241 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7242 uint32_t u32IntInfo = u8Interrupt
7243 | VMX_EXIT_INTERRUPTION_INFO_VALID
7244 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7245
7246 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7247 }
7248 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7249 {
7250 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7251 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7252 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7253
7254 /*
7255 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7256 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7257 * need to re-set this force-flag here.
7258 */
7259 }
7260 else
7261 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7262 }
7263 else
7264 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7265 }
7266
7267 return fIntrState;
7268}
7269
7270
7271/**
7272 * Sets a pending-debug exception to be delivered to the guest if the guest is
7273 * single-stepping in the VMCS.
7274 *
7275 * @param pVCpu The cross context virtual CPU structure.
7276 */
7277DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7278{
7279 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7280 RT_NOREF(pVCpu);
7281 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7282}
7283
7284
7285/**
7286 * Injects any pending events into the guest if the guest is in a state to
7287 * receive them.
7288 *
7289 * @returns Strict VBox status code (i.e. informational status codes too).
7290 * @param pVCpu The cross context virtual CPU structure.
7291 * @param fIntrState The VT-x guest-interruptibility state.
7292 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7293 * return VINF_EM_DBG_STEPPED if the event was
7294 * dispatched directly.
7295 */
7296static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7297{
7298 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7299 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7300
7301 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7302 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7303
7304 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7305 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7306 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7307 Assert(!TRPMHasTrap(pVCpu));
7308
7309 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7310 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7311 if (pVCpu->hm.s.Event.fPending)
7312 {
7313 /*
7314 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7315 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7316 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7317 *
7318 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7319 */
7320 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7321#ifdef VBOX_STRICT
7322 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7323 {
7324 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7325 Assert(!fBlockInt);
7326 Assert(!fBlockSti);
7327 Assert(!fBlockMovSS);
7328 }
7329 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7330 {
7331 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7332 Assert(!fBlockSti);
7333 Assert(!fBlockMovSS);
7334 Assert(!fBlockNmi);
7335 }
7336#endif
7337 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7338 uIntType));
7339 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7340 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7341 &fIntrState);
7342 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7343
7344 /* Update the interruptibility-state as it could have been changed by
7345 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7346 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7347 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7348
7349 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7350 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7351 else
7352 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7353 }
7354
7355 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7356 if ( fBlockSti
7357 || fBlockMovSS)
7358 {
7359 if (!pVCpu->hm.s.fSingleInstruction)
7360 {
7361 /*
7362 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7363 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7364 * See Intel spec. 27.3.4 "Saving Non-Register State".
7365 */
7366 Assert(!DBGFIsStepping(pVCpu));
7367 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7368 AssertRCReturn(rc, rc);
7369 if (pCtx->eflags.Bits.u1TF)
7370 {
7371 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7372 AssertRCReturn(rc2, rc2);
7373 }
7374 }
7375 else if (pCtx->eflags.Bits.u1TF)
7376 {
7377 /*
7378 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7379 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7380 */
7381 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7382 fIntrState = 0;
7383 }
7384 }
7385
7386 /*
7387 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7388 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7389 */
7390 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7391 AssertRCReturn(rc3, rc3);
7392
7393 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7394 NOREF(fBlockMovSS); NOREF(fBlockSti);
7395 return rcStrict;
7396}
7397
7398
7399/**
7400 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7401 *
7402 * @param pVCpu The cross context virtual CPU structure.
7403 */
7404DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7405{
7406 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7407 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7408}
7409
7410
7411/**
7412 * Injects a double-fault (\#DF) exception into the VM.
7413 *
7414 * @returns Strict VBox status code (i.e. informational status codes too).
7415 * @param pVCpu The cross context virtual CPU structure.
7416 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7417 * and should return VINF_EM_DBG_STEPPED if the event
7418 * is injected directly (register modified by us, not
7419 * by hardware on VM-entry).
7420 * @param pfIntrState Pointer to the current guest interruptibility-state.
7421 * This interruptibility-state will be updated if
7422 * necessary. This cannot not be NULL.
7423 */
7424DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7425{
7426 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7427 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7428 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7429 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7430 pfIntrState);
7431}
7432
7433
7434/**
7435 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7436 *
7437 * @param pVCpu The cross context virtual CPU structure.
7438 */
7439DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7440{
7441 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7442 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7443 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7444}
7445
7446
7447/**
7448 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7449 *
7450 * @param pVCpu The cross context virtual CPU structure.
7451 * @param cbInstr The value of RIP that is to be pushed on the guest
7452 * stack.
7453 */
7454DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7455{
7456 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7457 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7458 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7459}
7460
7461
7462/**
7463 * Injects a general-protection (\#GP) fault into the VM.
7464 *
7465 * @returns Strict VBox status code (i.e. informational status codes too).
7466 * @param pVCpu The cross context virtual CPU structure.
7467 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7468 * mode, i.e. in real-mode it's not valid).
7469 * @param u32ErrorCode The error code associated with the \#GP.
7470 * @param fStepping Whether we're running in
7471 * hmR0VmxRunGuestCodeStep() and should return
7472 * VINF_EM_DBG_STEPPED if the event is injected
7473 * directly (register modified by us, not by
7474 * hardware on VM-entry).
7475 * @param pfIntrState Pointer to the current guest interruptibility-state.
7476 * This interruptibility-state will be updated if
7477 * necessary. This cannot not be NULL.
7478 */
7479DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7480 uint32_t *pfIntrState)
7481{
7482 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7483 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7484 if (fErrorCodeValid)
7485 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7486 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7487 pfIntrState);
7488}
7489
7490
7491/**
7492 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7493 *
7494 * @param pVCpu The cross context virtual CPU structure.
7495 * @param uVector The software interrupt vector number.
7496 * @param cbInstr The value of RIP that is to be pushed on the guest
7497 * stack.
7498 */
7499DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7500{
7501 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7502 if ( uVector == X86_XCPT_BP
7503 || uVector == X86_XCPT_OF)
7504 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7505 else
7506 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7507 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7508}
7509
7510
7511/**
7512 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7513 * stack.
7514 *
7515 * @returns Strict VBox status code (i.e. informational status codes too).
7516 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7517 * @param pVCpu The cross context virtual CPU structure.
7518 * @param uValue The value to push to the guest stack.
7519 */
7520static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7521{
7522 /*
7523 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7524 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7525 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7526 */
7527 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7528 if (pCtx->sp == 1)
7529 return VINF_EM_RESET;
7530 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7531 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7532 AssertRC(rc);
7533 return rc;
7534}
7535
7536
7537/**
7538 * Injects an event into the guest upon VM-entry by updating the relevant fields
7539 * in the VM-entry area in the VMCS.
7540 *
7541 * @returns Strict VBox status code (i.e. informational status codes too).
7542 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7543 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7544 *
7545 * @param pVCpu The cross context virtual CPU structure.
7546 * @param u64IntInfo The VM-entry interruption-information field.
7547 * @param cbInstr The VM-entry instruction length in bytes (for
7548 * software interrupts, exceptions and privileged
7549 * software exceptions).
7550 * @param u32ErrCode The VM-entry exception error code.
7551 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7552 * @param pfIntrState Pointer to the current guest interruptibility-state.
7553 * This interruptibility-state will be updated if
7554 * necessary. This cannot not be NULL.
7555 * @param fStepping Whether we're running in
7556 * hmR0VmxRunGuestCodeStep() and should return
7557 * VINF_EM_DBG_STEPPED if the event is injected
7558 * directly (register modified by us, not by
7559 * hardware on VM-entry).
7560 */
7561static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7562 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7563{
7564 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7565 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7566 Assert(pfIntrState);
7567
7568 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7569 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7570 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7571 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7572
7573#ifdef VBOX_STRICT
7574 /*
7575 * Validate the error-code-valid bit for hardware exceptions.
7576 * No error codes for exceptions in real-mode.
7577 *
7578 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7579 */
7580 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7581 && !CPUMIsGuestInRealModeEx(pCtx))
7582 {
7583 switch (uVector)
7584 {
7585 case X86_XCPT_PF:
7586 case X86_XCPT_DF:
7587 case X86_XCPT_TS:
7588 case X86_XCPT_NP:
7589 case X86_XCPT_SS:
7590 case X86_XCPT_GP:
7591 case X86_XCPT_AC:
7592 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7593 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7594 RT_FALL_THRU();
7595 default:
7596 break;
7597 }
7598 }
7599#endif
7600
7601 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7602 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7603 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7604
7605 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7606
7607 /*
7608 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7609 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7610 * interrupt handler in the (real-mode) guest.
7611 *
7612 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7613 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7614 */
7615 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7616 {
7617 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7618 {
7619 /*
7620 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7621 * set the deliver-error-code bit.
7622 *
7623 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7624 */
7625 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7626 }
7627 else
7628 {
7629 PVM pVM = pVCpu->CTX_SUFF(pVM);
7630 Assert(PDMVmmDevHeapIsEnabled(pVM));
7631 Assert(pVM->hm.s.vmx.pRealModeTSS);
7632
7633 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7634 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7635 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7636 AssertRCReturn(rc2, rc2);
7637
7638 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7639 size_t const cbIdtEntry = sizeof(X86IDTR16);
7640 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7641 {
7642 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7643 if (uVector == X86_XCPT_DF)
7644 return VINF_EM_RESET;
7645
7646 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7647 if (uVector == X86_XCPT_GP)
7648 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7649
7650 /*
7651 * If we're injecting an event with no valid IDT entry, inject a #GP.
7652 * No error codes for exceptions in real-mode.
7653 *
7654 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7655 */
7656 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7657 }
7658
7659 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7660 uint16_t uGuestIp = pCtx->ip;
7661 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7662 {
7663 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7664 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7665 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7666 }
7667 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7668 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7669
7670 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7671 X86IDTR16 IdtEntry;
7672 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7673 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7674 AssertRCReturn(rc2, rc2);
7675
7676 /* Construct the stack frame for the interrupt/exception handler. */
7677 VBOXSTRICTRC rcStrict;
7678 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7679 if (rcStrict == VINF_SUCCESS)
7680 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7681 if (rcStrict == VINF_SUCCESS)
7682 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7683
7684 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7685 if (rcStrict == VINF_SUCCESS)
7686 {
7687 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7688 pCtx->rip = IdtEntry.offSel;
7689 pCtx->cs.Sel = IdtEntry.uSel;
7690 pCtx->cs.ValidSel = IdtEntry.uSel;
7691 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7692 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7693 && uVector == X86_XCPT_PF)
7694 pCtx->cr2 = GCPtrFaultAddress;
7695
7696 /* If any other guest-state bits are changed here, make sure to update
7697 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7698 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7699 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7700 | HM_CHANGED_GUEST_RSP);
7701
7702 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7703 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7704 {
7705 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7706 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7707 Log4Func(("Clearing inhibition due to STI\n"));
7708 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7709 }
7710 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7711 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7712
7713 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7714 it, if we are returning to ring-3 before executing guest code. */
7715 pVCpu->hm.s.Event.fPending = false;
7716
7717 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7718 if (fStepping)
7719 rcStrict = VINF_EM_DBG_STEPPED;
7720 }
7721 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7722 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7723 return rcStrict;
7724 }
7725 }
7726
7727 /* Validate. */
7728 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7729 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7730 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7731
7732 /* Inject. */
7733 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7734 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7735 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7736 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7737 AssertRCReturn(rc, rc);
7738
7739 /* Update CR2. */
7740 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7741 && uVector == X86_XCPT_PF)
7742 pCtx->cr2 = GCPtrFaultAddress;
7743
7744 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7745
7746 return VINF_SUCCESS;
7747}
7748
7749
7750/**
7751 * Clears the interrupt-window exiting control in the VMCS and if necessary
7752 * clears the current event in the VMCS as well.
7753 *
7754 * @returns VBox status code.
7755 * @param pVCpu The cross context virtual CPU structure.
7756 *
7757 * @remarks Use this function only to clear events that have not yet been
7758 * delivered to the guest but are injected in the VMCS!
7759 * @remarks No-long-jump zone!!!
7760 */
7761static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7762{
7763 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7764 {
7765 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7766 Log4Func(("Cleared interrupt widow\n"));
7767 }
7768
7769 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7770 {
7771 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7772 Log4Func(("Cleared interrupt widow\n"));
7773 }
7774}
7775
7776
7777/**
7778 * Enters the VT-x session.
7779 *
7780 * @returns VBox status code.
7781 * @param pVCpu The cross context virtual CPU structure.
7782 * @param pHostCpu Pointer to the global CPU info struct.
7783 */
7784VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7785{
7786 AssertPtr(pVCpu);
7787 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7788 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7789 RT_NOREF(pHostCpu);
7790
7791 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7792 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7793 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7794
7795#ifdef VBOX_STRICT
7796 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7797 RTCCUINTREG uHostCR4 = ASMGetCR4();
7798 if (!(uHostCR4 & X86_CR4_VMXE))
7799 {
7800 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7801 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7802 }
7803#endif
7804
7805 /*
7806 * Load the VCPU's VMCS as the current (and active) one.
7807 */
7808 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7809 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7810 if (RT_FAILURE(rc))
7811 return rc;
7812
7813 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7814 pVCpu->hm.s.fLeaveDone = false;
7815 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7816
7817 return VINF_SUCCESS;
7818}
7819
7820
7821/**
7822 * The thread-context callback (only on platforms which support it).
7823 *
7824 * @param enmEvent The thread-context event.
7825 * @param pVCpu The cross context virtual CPU structure.
7826 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7827 * @thread EMT(pVCpu)
7828 */
7829VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7830{
7831 NOREF(fGlobalInit);
7832
7833 switch (enmEvent)
7834 {
7835 case RTTHREADCTXEVENT_OUT:
7836 {
7837 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7838 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7839 VMCPU_ASSERT_EMT(pVCpu);
7840
7841 /* No longjmps (logger flushes, locks) in this fragile context. */
7842 VMMRZCallRing3Disable(pVCpu);
7843 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7844
7845 /*
7846 * Restore host-state (FPU, debug etc.)
7847 */
7848 if (!pVCpu->hm.s.fLeaveDone)
7849 {
7850 /*
7851 * Do -not- import the guest-state here as we might already be in the middle of importing
7852 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7853 */
7854 hmR0VmxLeave(pVCpu, false /* fImportState */);
7855 pVCpu->hm.s.fLeaveDone = true;
7856 }
7857
7858 /* Leave HM context, takes care of local init (term). */
7859 int rc = HMR0LeaveCpu(pVCpu);
7860 AssertRC(rc); NOREF(rc);
7861
7862 /* Restore longjmp state. */
7863 VMMRZCallRing3Enable(pVCpu);
7864 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7865 break;
7866 }
7867
7868 case RTTHREADCTXEVENT_IN:
7869 {
7870 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7871 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7872 VMCPU_ASSERT_EMT(pVCpu);
7873
7874 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7875 VMMRZCallRing3Disable(pVCpu);
7876 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7877
7878 /* Initialize the bare minimum state required for HM. This takes care of
7879 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7880 int rc = hmR0EnterCpu(pVCpu);
7881 AssertRC(rc);
7882 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7883 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7884
7885 /* Load the active VMCS as the current one. */
7886 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7887 {
7888 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7889 AssertRC(rc); NOREF(rc);
7890 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7891 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7892 }
7893 pVCpu->hm.s.fLeaveDone = false;
7894
7895 /* Restore longjmp state. */
7896 VMMRZCallRing3Enable(pVCpu);
7897 break;
7898 }
7899
7900 default:
7901 break;
7902 }
7903}
7904
7905
7906/**
7907 * Exports the host state into the VMCS host-state area.
7908 * Sets up the VM-exit MSR-load area.
7909 *
7910 * The CPU state will be loaded from these fields on every successful VM-exit.
7911 *
7912 * @returns VBox status code.
7913 * @param pVCpu The cross context virtual CPU structure.
7914 *
7915 * @remarks No-long-jump zone!!!
7916 */
7917static int hmR0VmxExportHostState(PVMCPU pVCpu)
7918{
7919 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7920
7921 int rc = VINF_SUCCESS;
7922 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7923 {
7924 rc = hmR0VmxExportHostControlRegs();
7925 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7926
7927 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7928 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7929
7930 rc = hmR0VmxExportHostMsrs(pVCpu);
7931 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7932
7933 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7934 }
7935 return rc;
7936}
7937
7938
7939/**
7940 * Saves the host state in the VMCS host-state.
7941 *
7942 * @returns VBox status code.
7943 * @param pVCpu The cross context virtual CPU structure.
7944 *
7945 * @remarks No-long-jump zone!!!
7946 */
7947VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7948{
7949 AssertPtr(pVCpu);
7950 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7951
7952 /*
7953 * Export the host state here while entering HM context.
7954 * When thread-context hooks are used, we might get preempted and have to re-save the host
7955 * state but most of the time we won't be, so do it here before we disable interrupts.
7956 */
7957 return hmR0VmxExportHostState(pVCpu);
7958}
7959
7960
7961/**
7962 * Exports the guest state into the VMCS guest-state area.
7963 *
7964 * The will typically be done before VM-entry when the guest-CPU state and the
7965 * VMCS state may potentially be out of sync.
7966 *
7967 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7968 * VM-entry controls.
7969 * Sets up the appropriate VMX non-root function to execute guest code based on
7970 * the guest CPU mode.
7971 *
7972 * @returns VBox strict status code.
7973 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7974 * without unrestricted guest access and the VMMDev is not presently
7975 * mapped (e.g. EFI32).
7976 *
7977 * @param pVCpu The cross context virtual CPU structure.
7978 *
7979 * @remarks No-long-jump zone!!!
7980 */
7981static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7982{
7983 AssertPtr(pVCpu);
7984 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7985
7986 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7987
7988 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7989
7990 /* Determine real-on-v86 mode. */
7991 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7992 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7993 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7994 {
7995 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7996 }
7997
7998 /*
7999 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8000 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8001 */
8002 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8003 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8004
8005 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8006 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8007 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8008
8009 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8010 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8011 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8012
8013 rc = hmR0VmxExportGuestCR0(pVCpu);
8014 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8015
8016 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8017 if (rcStrict == VINF_SUCCESS)
8018 { /* likely */ }
8019 else
8020 {
8021 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8022 return rcStrict;
8023 }
8024
8025 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8026 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8027
8028 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8029 may alter controls if we determine we don't have to swap EFER after all. */
8030 rc = hmR0VmxExportGuestMsrs(pVCpu);
8031 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8032
8033 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8034 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8035
8036 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8037 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8038
8039 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8040 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8041 rc = hmR0VmxExportGuestRip(pVCpu);
8042 rc |= hmR0VmxExportGuestRsp(pVCpu);
8043 rc |= hmR0VmxExportGuestRflags(pVCpu);
8044 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8045
8046 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8047 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8048 | HM_CHANGED_GUEST_CR2
8049 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8050 | HM_CHANGED_GUEST_X87
8051 | HM_CHANGED_GUEST_SSE_AVX
8052 | HM_CHANGED_GUEST_OTHER_XSAVE
8053 | HM_CHANGED_GUEST_XCRx
8054 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8055 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8056 | HM_CHANGED_GUEST_TSC_AUX
8057 | HM_CHANGED_GUEST_OTHER_MSRS
8058 | HM_CHANGED_GUEST_HWVIRT
8059 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8060
8061 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8062 return rc;
8063}
8064
8065
8066/**
8067 * Exports the state shared between the host and guest into the VMCS.
8068 *
8069 * @param pVCpu The cross context virtual CPU structure.
8070 *
8071 * @remarks No-long-jump zone!!!
8072 */
8073static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8074{
8075 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8076 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8077
8078 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8079 {
8080 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8081 AssertRC(rc);
8082 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8083
8084 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8085 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8086 {
8087 rc = hmR0VmxExportGuestRflags(pVCpu);
8088 AssertRC(rc);
8089 }
8090 }
8091
8092 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8093 {
8094 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8095 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8096 }
8097
8098 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8099 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8100}
8101
8102
8103/**
8104 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8105 *
8106 * @returns Strict VBox status code (i.e. informational status codes too).
8107 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8108 * without unrestricted guest access and the VMMDev is not presently
8109 * mapped (e.g. EFI32).
8110 *
8111 * @param pVCpu The cross context virtual CPU structure.
8112 *
8113 * @remarks No-long-jump zone!!!
8114 */
8115static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8116{
8117 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8118 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8119 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8120
8121#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8122 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8123#endif
8124
8125 /*
8126 * For many exits it's only RIP that changes and hence try to export it first
8127 * without going through a lot of change flag checks.
8128 */
8129 VBOXSTRICTRC rcStrict;
8130 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8131 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8132 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8133 {
8134 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8135 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8136 { /* likely */}
8137 else
8138 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8139 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8140 }
8141 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8142 {
8143 rcStrict = hmR0VmxExportGuestState(pVCpu);
8144 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8145 { /* likely */}
8146 else
8147 {
8148 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8149 VBOXSTRICTRC_VAL(rcStrict)));
8150 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8151 return rcStrict;
8152 }
8153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8154 }
8155 else
8156 rcStrict = VINF_SUCCESS;
8157
8158#ifdef VBOX_STRICT
8159 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8160 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8161 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8162 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8163 ("fCtxChanged=%#RX64\n", fCtxChanged));
8164#endif
8165 return rcStrict;
8166}
8167
8168
8169/**
8170 * Does the preparations before executing guest code in VT-x.
8171 *
8172 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8173 * recompiler/IEM. We must be cautious what we do here regarding committing
8174 * guest-state information into the VMCS assuming we assuredly execute the
8175 * guest in VT-x mode.
8176 *
8177 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8178 * the common-state (TRPM/forceflags), we must undo those changes so that the
8179 * recompiler/IEM can (and should) use them when it resumes guest execution.
8180 * Otherwise such operations must be done when we can no longer exit to ring-3.
8181 *
8182 * @returns Strict VBox status code (i.e. informational status codes too).
8183 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8184 * have been disabled.
8185 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8186 * double-fault into the guest.
8187 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8188 * dispatched directly.
8189 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8190 *
8191 * @param pVCpu The cross context virtual CPU structure.
8192 * @param pVmxTransient Pointer to the VMX transient structure.
8193 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8194 * us ignore some of the reasons for returning to
8195 * ring-3, and return VINF_EM_DBG_STEPPED if event
8196 * dispatching took place.
8197 */
8198static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8199{
8200 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8201
8202#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8203 PGMRZDynMapFlushAutoSet(pVCpu);
8204#endif
8205
8206 /* Check force flag actions that might require us to go back to ring-3. */
8207 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8208 if (rcStrict == VINF_SUCCESS)
8209 { /* FFs doesn't get set all the time. */ }
8210 else
8211 return rcStrict;
8212
8213 /*
8214 * Setup the virtualized-APIC accesses.
8215 *
8216 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8217 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8218 *
8219 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8220 */
8221 PVM pVM = pVCpu->CTX_SUFF(pVM);
8222 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8223 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8224 && PDMHasApic(pVM))
8225 {
8226 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8227 Assert(u64MsrApicBase);
8228 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8229
8230 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8231
8232 /* Unalias any existing mapping. */
8233 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8234 AssertRCReturn(rc, rc);
8235
8236 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8237 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8238 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8239 AssertRCReturn(rc, rc);
8240
8241 /* Update the per-VCPU cache of the APIC base MSR. */
8242 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8243 }
8244
8245 if (TRPMHasTrap(pVCpu))
8246 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8247 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8248
8249 /*
8250 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8251 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8252 * also result in triple-faulting the VM.
8253 */
8254 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8255 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8256 { /* likely */ }
8257 else
8258 {
8259 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8260 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8261 return rcStrict;
8262 }
8263
8264 /*
8265 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8266 * import CR3 themselves. We will need to update them here as even as late as the above
8267 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8268 * the below force flags to be set.
8269 */
8270 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8271 {
8272 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8273 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8274 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8275 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8276 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8277 }
8278 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8279 {
8280 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8281 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8282 }
8283
8284 /*
8285 * No longjmps to ring-3 from this point on!!!
8286 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8287 * This also disables flushing of the R0-logger instance (if any).
8288 */
8289 VMMRZCallRing3Disable(pVCpu);
8290
8291 /*
8292 * Export the guest state bits.
8293 *
8294 * We cannot perform longjmps while loading the guest state because we do not preserve the
8295 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8296 * CPU migration.
8297 *
8298 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8299 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8300 * Hence, loading of the guest state needs to be done -after- injection of events.
8301 */
8302 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8303 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8304 { /* likely */ }
8305 else
8306 {
8307 VMMRZCallRing3Enable(pVCpu);
8308 return rcStrict;
8309 }
8310
8311 /*
8312 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8313 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8314 * preemption disabled for a while. Since this is purly to aid the
8315 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8316 * disable interrupt on NT.
8317 *
8318 * We need to check for force-flags that could've possible been altered since we last
8319 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8320 * see @bugref{6398}).
8321 *
8322 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8323 * to ring-3 before executing guest code.
8324 */
8325 pVmxTransient->fEFlags = ASMIntDisableFlags();
8326
8327 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8328 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8329 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8330 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8331 {
8332 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8333 {
8334 pVCpu->hm.s.Event.fPending = false;
8335
8336 /*
8337 * We've injected any pending events. This is really the point of no return (to ring-3).
8338 *
8339 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8340 * returns from this function, so don't enable them here.
8341 */
8342 return VINF_SUCCESS;
8343 }
8344
8345 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8346 rcStrict = VINF_EM_RAW_INTERRUPT;
8347 }
8348 else
8349 {
8350 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8351 rcStrict = VINF_EM_RAW_TO_R3;
8352 }
8353
8354 ASMSetFlags(pVmxTransient->fEFlags);
8355 VMMRZCallRing3Enable(pVCpu);
8356
8357 return rcStrict;
8358}
8359
8360
8361/**
8362 * Prepares to run guest code in VT-x and we've committed to doing so. This
8363 * means there is no backing out to ring-3 or anywhere else at this
8364 * point.
8365 *
8366 * @param pVCpu The cross context virtual CPU structure.
8367 * @param pVmxTransient Pointer to the VMX transient structure.
8368 *
8369 * @remarks Called with preemption disabled.
8370 * @remarks No-long-jump zone!!!
8371 */
8372static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8373{
8374 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8375 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8376 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8377
8378 /*
8379 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8380 */
8381 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8382 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8383
8384 PVM pVM = pVCpu->CTX_SUFF(pVM);
8385 if (!CPUMIsGuestFPUStateActive(pVCpu))
8386 {
8387 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8388 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8389 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8390 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8391 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8392 }
8393
8394 /*
8395 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8396 */
8397 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8398 && pVCpu->hm.s.vmx.cMsrs > 0)
8399 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8400
8401 /*
8402 * Re-save the host state bits as we may've been preempted (only happens when
8403 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8404 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8405 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8406 * See @bugref{8432}.
8407 */
8408 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8409 {
8410 int rc = hmR0VmxExportHostState(pVCpu);
8411 AssertRC(rc);
8412 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8413 }
8414 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8415
8416 /*
8417 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8418 */
8419 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8420 hmR0VmxExportSharedState(pVCpu);
8421 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8422
8423 /* Store status of the shared guest-host state at the time of VM-entry. */
8424#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8425 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8426 {
8427 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8428 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8429 }
8430 else
8431#endif
8432 {
8433 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8434 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8435 }
8436
8437 /*
8438 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8439 */
8440 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8441 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8442
8443 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8444 RTCPUID idCurrentCpu = pCpu->idCpu;
8445 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8446 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8447 {
8448 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8449 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8450 }
8451
8452 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8453 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8454 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8455 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8456
8457 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8458
8459 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8460 to start executing. */
8461
8462 /*
8463 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8464 */
8465 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8466 {
8467 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8468 {
8469 bool fMsrUpdated;
8470 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8471 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8472 &fMsrUpdated);
8473 AssertRC(rc2);
8474 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8475 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8476 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8477 }
8478 else
8479 {
8480 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8481 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8482 }
8483 }
8484
8485 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8486 {
8487 bool fMsrUpdated;
8488 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8489 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8490 &fMsrUpdated);
8491 AssertRC(rc2);
8492 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8493 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8494 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8495 }
8496
8497#ifdef VBOX_STRICT
8498 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8499 hmR0VmxCheckHostEferMsr(pVCpu);
8500 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8501#endif
8502#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8503 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8504 {
8505 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8506 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8507 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8508 }
8509#endif
8510}
8511
8512
8513/**
8514 * Performs some essential restoration of state after running guest code in
8515 * VT-x.
8516 *
8517 * @param pVCpu The cross context virtual CPU structure.
8518 * @param pVmxTransient Pointer to the VMX transient structure.
8519 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8520 *
8521 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8522 *
8523 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8524 * unconditionally when it is safe to do so.
8525 */
8526static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8527{
8528 uint64_t const uHostTsc = ASMReadTSC();
8529 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8530
8531 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8532 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8533 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8534 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8535 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8536 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8537
8538 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8539 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8540
8541 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8542 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8543 Assert(!ASMIntAreEnabled());
8544 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8545
8546#if HC_ARCH_BITS == 64
8547 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8548#endif
8549#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8550 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8551 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8552 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8553#else
8554 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8555#endif
8556#ifdef VBOX_STRICT
8557 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8558#endif
8559 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8560
8561 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8562 uint32_t uExitReason;
8563 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8564 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8565 AssertRC(rc);
8566 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8567 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8568
8569 if (rcVMRun == VINF_SUCCESS)
8570 {
8571 /*
8572 * Update the VM-exit history array here even if the VM-entry failed due to:
8573 * - Invalid guest state.
8574 * - MSR loading.
8575 * - Machine-check event.
8576 *
8577 * In any of the above cases we will still have a "valid" VM-exit reason
8578 * despite @a fVMEntryFailed being false.
8579 *
8580 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8581 *
8582 * Note! We don't have CS or RIP at this point. Will probably address that later
8583 * by amending the history entry added here.
8584 */
8585 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8586 UINT64_MAX, uHostTsc);
8587
8588 if (!pVmxTransient->fVMEntryFailed)
8589 {
8590 VMMRZCallRing3Enable(pVCpu);
8591
8592 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8593 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8594
8595#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8596 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8597 AssertRC(rc);
8598#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8599 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8600 AssertRC(rc);
8601#else
8602 /*
8603 * Import the guest-interruptibility state always as we need it while evaluating
8604 * injecting events on re-entry.
8605 *
8606 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8607 * checking for real-mode while exporting the state because all bits that cause
8608 * mode changes wrt CR0 are intercepted.
8609 */
8610 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8611 AssertRC(rc);
8612#endif
8613
8614 /*
8615 * Sync the TPR shadow with our APIC state.
8616 */
8617 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8618 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8619 {
8620 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8621 AssertRC(rc);
8622 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8623 }
8624
8625 return;
8626 }
8627 }
8628 else
8629 {
8630 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8631 }
8632
8633 VMMRZCallRing3Enable(pVCpu);
8634}
8635
8636
8637/**
8638 * Runs the guest code using VT-x the normal way.
8639 *
8640 * @returns VBox status code.
8641 * @param pVCpu The cross context virtual CPU structure.
8642 *
8643 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8644 */
8645static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8646{
8647 VMXTRANSIENT VmxTransient;
8648 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8649 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8650 uint32_t cLoops = 0;
8651
8652 for (;; cLoops++)
8653 {
8654 Assert(!HMR0SuspendPending());
8655 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8656
8657 /* Preparatory work for running guest code, this may force us to return
8658 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8659 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8660 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8661 if (rcStrict != VINF_SUCCESS)
8662 break;
8663
8664 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8665 int rcRun = hmR0VmxRunGuest(pVCpu);
8666
8667 /* Restore any residual host-state and save any bits shared between host
8668 and guest into the guest-CPU state. Re-enables interrupts! */
8669 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8670
8671 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8672 if (RT_SUCCESS(rcRun))
8673 { /* very likely */ }
8674 else
8675 {
8676 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8677 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8678 return rcRun;
8679 }
8680
8681 /* Profile the VM-exit. */
8682 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8683 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8684 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8685 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8686 HMVMX_START_EXIT_DISPATCH_PROF();
8687
8688 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8689
8690 /* Handle the VM-exit. */
8691#ifdef HMVMX_USE_FUNCTION_TABLE
8692 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8693#else
8694 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8695#endif
8696 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8697 if (rcStrict == VINF_SUCCESS)
8698 {
8699 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8700 continue; /* likely */
8701 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8702 rcStrict = VINF_EM_RAW_INTERRUPT;
8703 }
8704 break;
8705 }
8706
8707 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8708 return rcStrict;
8709}
8710
8711
8712
8713/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8714 * probes.
8715 *
8716 * The following few functions and associated structure contains the bloat
8717 * necessary for providing detailed debug events and dtrace probes as well as
8718 * reliable host side single stepping. This works on the principle of
8719 * "subclassing" the normal execution loop and workers. We replace the loop
8720 * method completely and override selected helpers to add necessary adjustments
8721 * to their core operation.
8722 *
8723 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8724 * any performance for debug and analysis features.
8725 *
8726 * @{
8727 */
8728
8729/**
8730 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8731 * the debug run loop.
8732 */
8733typedef struct VMXRUNDBGSTATE
8734{
8735 /** The RIP we started executing at. This is for detecting that we stepped. */
8736 uint64_t uRipStart;
8737 /** The CS we started executing with. */
8738 uint16_t uCsStart;
8739
8740 /** Whether we've actually modified the 1st execution control field. */
8741 bool fModifiedProcCtls : 1;
8742 /** Whether we've actually modified the 2nd execution control field. */
8743 bool fModifiedProcCtls2 : 1;
8744 /** Whether we've actually modified the exception bitmap. */
8745 bool fModifiedXcptBitmap : 1;
8746
8747 /** We desire the modified the CR0 mask to be cleared. */
8748 bool fClearCr0Mask : 1;
8749 /** We desire the modified the CR4 mask to be cleared. */
8750 bool fClearCr4Mask : 1;
8751 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8752 uint32_t fCpe1Extra;
8753 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8754 uint32_t fCpe1Unwanted;
8755 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8756 uint32_t fCpe2Extra;
8757 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8758 uint32_t bmXcptExtra;
8759 /** The sequence number of the Dtrace provider settings the state was
8760 * configured against. */
8761 uint32_t uDtraceSettingsSeqNo;
8762 /** VM-exits to check (one bit per VM-exit). */
8763 uint32_t bmExitsToCheck[3];
8764
8765 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8766 uint32_t fProcCtlsInitial;
8767 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8768 uint32_t fProcCtls2Initial;
8769 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8770 uint32_t bmXcptInitial;
8771} VMXRUNDBGSTATE;
8772AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8773typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8774
8775
8776/**
8777 * Initializes the VMXRUNDBGSTATE structure.
8778 *
8779 * @param pVCpu The cross context virtual CPU structure of the
8780 * calling EMT.
8781 * @param pDbgState The structure to initialize.
8782 */
8783static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8784{
8785 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8786 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8787
8788 pDbgState->fModifiedProcCtls = false;
8789 pDbgState->fModifiedProcCtls2 = false;
8790 pDbgState->fModifiedXcptBitmap = false;
8791 pDbgState->fClearCr0Mask = false;
8792 pDbgState->fClearCr4Mask = false;
8793 pDbgState->fCpe1Extra = 0;
8794 pDbgState->fCpe1Unwanted = 0;
8795 pDbgState->fCpe2Extra = 0;
8796 pDbgState->bmXcptExtra = 0;
8797 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8798 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8799 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8800}
8801
8802
8803/**
8804 * Updates the VMSC fields with changes requested by @a pDbgState.
8805 *
8806 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8807 * immediately before executing guest code, i.e. when interrupts are disabled.
8808 * We don't check status codes here as we cannot easily assert or return in the
8809 * latter case.
8810 *
8811 * @param pVCpu The cross context virtual CPU structure.
8812 * @param pDbgState The debug state.
8813 */
8814static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8815{
8816 /*
8817 * Ensure desired flags in VMCS control fields are set.
8818 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8819 *
8820 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8821 * there should be no stale data in pCtx at this point.
8822 */
8823 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8824 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8825 {
8826 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8827 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8828 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8829 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8830 pDbgState->fModifiedProcCtls = true;
8831 }
8832
8833 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8834 {
8835 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8836 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8837 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8838 pDbgState->fModifiedProcCtls2 = true;
8839 }
8840
8841 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8842 {
8843 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8844 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8845 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8846 pDbgState->fModifiedXcptBitmap = true;
8847 }
8848
8849 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8850 {
8851 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8852 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8853 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8854 }
8855
8856 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8857 {
8858 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8859 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8860 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8861 }
8862}
8863
8864
8865static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8866{
8867 /*
8868 * Restore VM-exit control settings as we may not reenter this function the
8869 * next time around.
8870 */
8871 /* We reload the initial value, trigger what we can of recalculations the
8872 next time around. From the looks of things, that's all that's required atm. */
8873 if (pDbgState->fModifiedProcCtls)
8874 {
8875 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8876 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8877 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8878 AssertRCReturn(rc2, rc2);
8879 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8880 }
8881
8882 /* We're currently the only ones messing with this one, so just restore the
8883 cached value and reload the field. */
8884 if ( pDbgState->fModifiedProcCtls2
8885 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8886 {
8887 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8888 AssertRCReturn(rc2, rc2);
8889 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8890 }
8891
8892 /* If we've modified the exception bitmap, we restore it and trigger
8893 reloading and partial recalculation the next time around. */
8894 if (pDbgState->fModifiedXcptBitmap)
8895 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8896
8897 return rcStrict;
8898}
8899
8900
8901/**
8902 * Configures VM-exit controls for current DBGF and DTrace settings.
8903 *
8904 * This updates @a pDbgState and the VMCS execution control fields to reflect
8905 * the necessary VM-exits demanded by DBGF and DTrace.
8906 *
8907 * @param pVCpu The cross context virtual CPU structure.
8908 * @param pDbgState The debug state.
8909 * @param pVmxTransient Pointer to the VMX transient structure. May update
8910 * fUpdateTscOffsettingAndPreemptTimer.
8911 */
8912static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8913{
8914 /*
8915 * Take down the dtrace serial number so we can spot changes.
8916 */
8917 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8918 ASMCompilerBarrier();
8919
8920 /*
8921 * We'll rebuild most of the middle block of data members (holding the
8922 * current settings) as we go along here, so start by clearing it all.
8923 */
8924 pDbgState->bmXcptExtra = 0;
8925 pDbgState->fCpe1Extra = 0;
8926 pDbgState->fCpe1Unwanted = 0;
8927 pDbgState->fCpe2Extra = 0;
8928 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8929 pDbgState->bmExitsToCheck[i] = 0;
8930
8931 /*
8932 * Software interrupts (INT XXh) - no idea how to trigger these...
8933 */
8934 PVM pVM = pVCpu->CTX_SUFF(pVM);
8935 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8936 || VBOXVMM_INT_SOFTWARE_ENABLED())
8937 {
8938 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8939 }
8940
8941 /*
8942 * INT3 breakpoints - triggered by #BP exceptions.
8943 */
8944 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8945 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8946
8947 /*
8948 * Exception bitmap and XCPT events+probes.
8949 */
8950 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8951 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8952 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8953
8954 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8955 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8956 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8957 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8958 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8959 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8960 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8961 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8962 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8963 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8964 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8965 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8966 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8967 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8968 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8969 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8970 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8971 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8972
8973 if (pDbgState->bmXcptExtra)
8974 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8975
8976 /*
8977 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8978 *
8979 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8980 * So, when adding/changing/removing please don't forget to update it.
8981 *
8982 * Some of the macros are picking up local variables to save horizontal space,
8983 * (being able to see it in a table is the lesser evil here).
8984 */
8985#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8986 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8987 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8988#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8989 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8990 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8991 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8992 } else do { } while (0)
8993#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8994 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8995 { \
8996 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8997 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8998 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8999 } else do { } while (0)
9000#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9001 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9002 { \
9003 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9004 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9005 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9006 } else do { } while (0)
9007#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9008 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9009 { \
9010 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9011 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9012 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9013 } else do { } while (0)
9014
9015 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9016 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9017 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9018 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9019 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9020
9021 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9022 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9023 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9024 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9025 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9026 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9027 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9028 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9029 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9030 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9031 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9032 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9033 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9034 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9035 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9036 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9037 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9038 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9039 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9040 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9041 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9042 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9043 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9044 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9045 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9046 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9047 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9048 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9049 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9050 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9051 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9052 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9053 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9054 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9055 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9056 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9057
9058 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9059 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9060 {
9061 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9062 AssertRC(rc);
9063
9064#if 0 /** @todo fix me */
9065 pDbgState->fClearCr0Mask = true;
9066 pDbgState->fClearCr4Mask = true;
9067#endif
9068 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9069 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9070 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9071 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9072 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9073 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9074 require clearing here and in the loop if we start using it. */
9075 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9076 }
9077 else
9078 {
9079 if (pDbgState->fClearCr0Mask)
9080 {
9081 pDbgState->fClearCr0Mask = false;
9082 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9083 }
9084 if (pDbgState->fClearCr4Mask)
9085 {
9086 pDbgState->fClearCr4Mask = false;
9087 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9088 }
9089 }
9090 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9091 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9092
9093 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9094 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9095 {
9096 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9097 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9098 }
9099 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9101
9102 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9104 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9105 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9106 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9107 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9108 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9109 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9110#if 0 /** @todo too slow, fix handler. */
9111 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9112#endif
9113 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9114
9115 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9116 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9117 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9118 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9119 {
9120 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9121 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9122 }
9123 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9125 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9127
9128 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9129 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9130 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9131 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9132 {
9133 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9134 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9135 }
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9140
9141 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9143 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9145 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9147 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9149 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9151 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9153 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9155 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9157 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9159 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9160 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9161 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9163
9164#undef IS_EITHER_ENABLED
9165#undef SET_ONLY_XBM_IF_EITHER_EN
9166#undef SET_CPE1_XBM_IF_EITHER_EN
9167#undef SET_CPEU_XBM_IF_EITHER_EN
9168#undef SET_CPE2_XBM_IF_EITHER_EN
9169
9170 /*
9171 * Sanitize the control stuff.
9172 */
9173 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9174 if (pDbgState->fCpe2Extra)
9175 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9176 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9177 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9178 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9179 {
9180 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9181 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9182 }
9183
9184 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9185 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9186 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9187 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9188}
9189
9190
9191/**
9192 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9193 * appropriate.
9194 *
9195 * The caller has checked the VM-exit against the
9196 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9197 * already, so we don't have to do that either.
9198 *
9199 * @returns Strict VBox status code (i.e. informational status codes too).
9200 * @param pVCpu The cross context virtual CPU structure.
9201 * @param pVmxTransient Pointer to the VMX-transient structure.
9202 * @param uExitReason The VM-exit reason.
9203 *
9204 * @remarks The name of this function is displayed by dtrace, so keep it short
9205 * and to the point. No longer than 33 chars long, please.
9206 */
9207static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9208{
9209 /*
9210 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9211 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9212 *
9213 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9214 * does. Must add/change/remove both places. Same ordering, please.
9215 *
9216 * Added/removed events must also be reflected in the next section
9217 * where we dispatch dtrace events.
9218 */
9219 bool fDtrace1 = false;
9220 bool fDtrace2 = false;
9221 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9222 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9223 uint32_t uEventArg = 0;
9224#define SET_EXIT(a_EventSubName) \
9225 do { \
9226 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9227 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9228 } while (0)
9229#define SET_BOTH(a_EventSubName) \
9230 do { \
9231 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9232 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9233 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9234 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9235 } while (0)
9236 switch (uExitReason)
9237 {
9238 case VMX_EXIT_MTF:
9239 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9240
9241 case VMX_EXIT_XCPT_OR_NMI:
9242 {
9243 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9244 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9245 {
9246 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9247 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9248 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9249 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9250 {
9251 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9252 {
9253 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9254 uEventArg = pVmxTransient->uExitIntErrorCode;
9255 }
9256 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9257 switch (enmEvent1)
9258 {
9259 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9260 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9261 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9262 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9263 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9264 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9265 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9266 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9267 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9268 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9269 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9270 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9271 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9272 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9273 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9274 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9275 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9276 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9277 default: break;
9278 }
9279 }
9280 else
9281 AssertFailed();
9282 break;
9283
9284 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9285 uEventArg = idxVector;
9286 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9287 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9288 break;
9289 }
9290 break;
9291 }
9292
9293 case VMX_EXIT_TRIPLE_FAULT:
9294 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9295 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9296 break;
9297 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9298 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9299 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9300 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9301 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9302
9303 /* Instruction specific VM-exits: */
9304 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9305 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9306 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9307 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9308 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9309 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9310 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9311 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9312 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9313 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9314 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9315 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9316 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9317 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9318 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9319 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9320 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9321 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9322 case VMX_EXIT_MOV_CRX:
9323 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9324 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9325 SET_BOTH(CRX_READ);
9326 else
9327 SET_BOTH(CRX_WRITE);
9328 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9329 break;
9330 case VMX_EXIT_MOV_DRX:
9331 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9332 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9333 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9334 SET_BOTH(DRX_READ);
9335 else
9336 SET_BOTH(DRX_WRITE);
9337 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9338 break;
9339 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9340 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9341 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9342 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9343 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9344 case VMX_EXIT_XDTR_ACCESS:
9345 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9346 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9347 {
9348 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9349 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9350 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9351 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9352 }
9353 break;
9354
9355 case VMX_EXIT_TR_ACCESS:
9356 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9357 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9358 {
9359 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9360 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9361 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9362 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9363 }
9364 break;
9365
9366 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9367 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9368 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9369 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9370 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9371 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9372 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9373 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9374 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9375 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9376 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9377
9378 /* Events that aren't relevant at this point. */
9379 case VMX_EXIT_EXT_INT:
9380 case VMX_EXIT_INT_WINDOW:
9381 case VMX_EXIT_NMI_WINDOW:
9382 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9383 case VMX_EXIT_PREEMPT_TIMER:
9384 case VMX_EXIT_IO_INSTR:
9385 break;
9386
9387 /* Errors and unexpected events. */
9388 case VMX_EXIT_INIT_SIGNAL:
9389 case VMX_EXIT_SIPI:
9390 case VMX_EXIT_IO_SMI:
9391 case VMX_EXIT_SMI:
9392 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9393 case VMX_EXIT_ERR_MSR_LOAD:
9394 case VMX_EXIT_ERR_MACHINE_CHECK:
9395 break;
9396
9397 default:
9398 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9399 break;
9400 }
9401#undef SET_BOTH
9402#undef SET_EXIT
9403
9404 /*
9405 * Dtrace tracepoints go first. We do them here at once so we don't
9406 * have to copy the guest state saving and stuff a few dozen times.
9407 * Down side is that we've got to repeat the switch, though this time
9408 * we use enmEvent since the probes are a subset of what DBGF does.
9409 */
9410 if (fDtrace1 || fDtrace2)
9411 {
9412 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9413 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9414 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9415 switch (enmEvent1)
9416 {
9417 /** @todo consider which extra parameters would be helpful for each probe. */
9418 case DBGFEVENT_END: break;
9419 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9420 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9421 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9422 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9424 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9426 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9427 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9428 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9429 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9430 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9431 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9432 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9433 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9434 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9435 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9436 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9437 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9438 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9439 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9440 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9445 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9446 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9447 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9448 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9449 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9450 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9451 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9452 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9453 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9454 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9483 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9484 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9485 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9486 }
9487 switch (enmEvent2)
9488 {
9489 /** @todo consider which extra parameters would be helpful for each probe. */
9490 case DBGFEVENT_END: break;
9491 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9492 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9493 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9494 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9499 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9500 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9503 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9504 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9505 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9506 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9507 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9508 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9541 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9542 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9543 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9544 }
9545 }
9546
9547 /*
9548 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9549 * the DBGF call will do a full check).
9550 *
9551 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9552 * Note! If we have to events, we prioritize the first, i.e. the instruction
9553 * one, in order to avoid event nesting.
9554 */
9555 PVM pVM = pVCpu->CTX_SUFF(pVM);
9556 if ( enmEvent1 != DBGFEVENT_END
9557 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9558 {
9559 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9560 if (rcStrict != VINF_SUCCESS)
9561 return rcStrict;
9562 }
9563 else if ( enmEvent2 != DBGFEVENT_END
9564 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9565 {
9566 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9567 if (rcStrict != VINF_SUCCESS)
9568 return rcStrict;
9569 }
9570
9571 return VINF_SUCCESS;
9572}
9573
9574
9575/**
9576 * Single-stepping VM-exit filtering.
9577 *
9578 * This is preprocessing the VM-exits and deciding whether we've gotten far
9579 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9580 * handling is performed.
9581 *
9582 * @returns Strict VBox status code (i.e. informational status codes too).
9583 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9584 * @param pVmxTransient Pointer to the VMX-transient structure.
9585 * @param pDbgState The debug state.
9586 */
9587DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9588{
9589 /*
9590 * Expensive (saves context) generic dtrace VM-exit probe.
9591 */
9592 uint32_t const uExitReason = pVmxTransient->uExitReason;
9593 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9594 { /* more likely */ }
9595 else
9596 {
9597 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9598 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9599 AssertRC(rc);
9600 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9601 }
9602
9603 /*
9604 * Check for host NMI, just to get that out of the way.
9605 */
9606 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9607 { /* normally likely */ }
9608 else
9609 {
9610 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9611 AssertRCReturn(rc2, rc2);
9612 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9613 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9614 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9615 }
9616
9617 /*
9618 * Check for single stepping event if we're stepping.
9619 */
9620 if (pVCpu->hm.s.fSingleInstruction)
9621 {
9622 switch (uExitReason)
9623 {
9624 case VMX_EXIT_MTF:
9625 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9626
9627 /* Various events: */
9628 case VMX_EXIT_XCPT_OR_NMI:
9629 case VMX_EXIT_EXT_INT:
9630 case VMX_EXIT_TRIPLE_FAULT:
9631 case VMX_EXIT_INT_WINDOW:
9632 case VMX_EXIT_NMI_WINDOW:
9633 case VMX_EXIT_TASK_SWITCH:
9634 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9635 case VMX_EXIT_APIC_ACCESS:
9636 case VMX_EXIT_EPT_VIOLATION:
9637 case VMX_EXIT_EPT_MISCONFIG:
9638 case VMX_EXIT_PREEMPT_TIMER:
9639
9640 /* Instruction specific VM-exits: */
9641 case VMX_EXIT_CPUID:
9642 case VMX_EXIT_GETSEC:
9643 case VMX_EXIT_HLT:
9644 case VMX_EXIT_INVD:
9645 case VMX_EXIT_INVLPG:
9646 case VMX_EXIT_RDPMC:
9647 case VMX_EXIT_RDTSC:
9648 case VMX_EXIT_RSM:
9649 case VMX_EXIT_VMCALL:
9650 case VMX_EXIT_VMCLEAR:
9651 case VMX_EXIT_VMLAUNCH:
9652 case VMX_EXIT_VMPTRLD:
9653 case VMX_EXIT_VMPTRST:
9654 case VMX_EXIT_VMREAD:
9655 case VMX_EXIT_VMRESUME:
9656 case VMX_EXIT_VMWRITE:
9657 case VMX_EXIT_VMXOFF:
9658 case VMX_EXIT_VMXON:
9659 case VMX_EXIT_MOV_CRX:
9660 case VMX_EXIT_MOV_DRX:
9661 case VMX_EXIT_IO_INSTR:
9662 case VMX_EXIT_RDMSR:
9663 case VMX_EXIT_WRMSR:
9664 case VMX_EXIT_MWAIT:
9665 case VMX_EXIT_MONITOR:
9666 case VMX_EXIT_PAUSE:
9667 case VMX_EXIT_XDTR_ACCESS:
9668 case VMX_EXIT_TR_ACCESS:
9669 case VMX_EXIT_INVEPT:
9670 case VMX_EXIT_RDTSCP:
9671 case VMX_EXIT_INVVPID:
9672 case VMX_EXIT_WBINVD:
9673 case VMX_EXIT_XSETBV:
9674 case VMX_EXIT_RDRAND:
9675 case VMX_EXIT_INVPCID:
9676 case VMX_EXIT_VMFUNC:
9677 case VMX_EXIT_RDSEED:
9678 case VMX_EXIT_XSAVES:
9679 case VMX_EXIT_XRSTORS:
9680 {
9681 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9682 AssertRCReturn(rc, rc);
9683 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9684 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9685 return VINF_EM_DBG_STEPPED;
9686 break;
9687 }
9688
9689 /* Errors and unexpected events: */
9690 case VMX_EXIT_INIT_SIGNAL:
9691 case VMX_EXIT_SIPI:
9692 case VMX_EXIT_IO_SMI:
9693 case VMX_EXIT_SMI:
9694 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9695 case VMX_EXIT_ERR_MSR_LOAD:
9696 case VMX_EXIT_ERR_MACHINE_CHECK:
9697 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9698 break;
9699
9700 default:
9701 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9702 break;
9703 }
9704 }
9705
9706 /*
9707 * Check for debugger event breakpoints and dtrace probes.
9708 */
9709 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9710 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9711 {
9712 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9713 if (rcStrict != VINF_SUCCESS)
9714 return rcStrict;
9715 }
9716
9717 /*
9718 * Normal processing.
9719 */
9720#ifdef HMVMX_USE_FUNCTION_TABLE
9721 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9722#else
9723 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9724#endif
9725}
9726
9727
9728/**
9729 * Single steps guest code using VT-x.
9730 *
9731 * @returns Strict VBox status code (i.e. informational status codes too).
9732 * @param pVCpu The cross context virtual CPU structure.
9733 *
9734 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9735 */
9736static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9737{
9738 VMXTRANSIENT VmxTransient;
9739 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9740
9741 /* Set HMCPU indicators. */
9742 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9743 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9744 pVCpu->hm.s.fDebugWantRdTscExit = false;
9745 pVCpu->hm.s.fUsingDebugLoop = true;
9746
9747 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9748 VMXRUNDBGSTATE DbgState;
9749 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9750 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9751
9752 /*
9753 * The loop.
9754 */
9755 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9756 for (uint32_t cLoops = 0; ; cLoops++)
9757 {
9758 Assert(!HMR0SuspendPending());
9759 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9760 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9761
9762 /*
9763 * Preparatory work for running guest code, this may force us to return
9764 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9765 */
9766 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9767 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9768 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9769 if (rcStrict != VINF_SUCCESS)
9770 break;
9771
9772 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9773 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9774
9775 /*
9776 * Now we can run the guest code.
9777 */
9778 int rcRun = hmR0VmxRunGuest(pVCpu);
9779
9780 /*
9781 * Restore any residual host-state and save any bits shared between host
9782 * and guest into the guest-CPU state. Re-enables interrupts!
9783 */
9784 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9785
9786 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9787 if (RT_SUCCESS(rcRun))
9788 { /* very likely */ }
9789 else
9790 {
9791 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9792 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9793 return rcRun;
9794 }
9795
9796 /* Profile the VM-exit. */
9797 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9799 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9800 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9801 HMVMX_START_EXIT_DISPATCH_PROF();
9802
9803 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9804
9805 /*
9806 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9807 */
9808 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9809 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9810 if (rcStrict != VINF_SUCCESS)
9811 break;
9812 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9813 {
9814 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9815 rcStrict = VINF_EM_RAW_INTERRUPT;
9816 break;
9817 }
9818
9819 /*
9820 * Stepping: Did the RIP change, if so, consider it a single step.
9821 * Otherwise, make sure one of the TFs gets set.
9822 */
9823 if (fStepping)
9824 {
9825 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9826 AssertRC(rc);
9827 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9828 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9829 {
9830 rcStrict = VINF_EM_DBG_STEPPED;
9831 break;
9832 }
9833 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9834 }
9835
9836 /*
9837 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9838 */
9839 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9840 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9841 }
9842
9843 /*
9844 * Clear the X86_EFL_TF if necessary.
9845 */
9846 if (pVCpu->hm.s.fClearTrapFlag)
9847 {
9848 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9849 AssertRC(rc);
9850 pVCpu->hm.s.fClearTrapFlag = false;
9851 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9852 }
9853 /** @todo there seems to be issues with the resume flag when the monitor trap
9854 * flag is pending without being used. Seen early in bios init when
9855 * accessing APIC page in protected mode. */
9856
9857 /*
9858 * Restore VM-exit control settings as we may not reenter this function the
9859 * next time around.
9860 */
9861 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9862
9863 /* Restore HMCPU indicators. */
9864 pVCpu->hm.s.fUsingDebugLoop = false;
9865 pVCpu->hm.s.fDebugWantRdTscExit = false;
9866 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9867
9868 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9869 return rcStrict;
9870}
9871
9872
9873/** @} */
9874
9875
9876/**
9877 * Checks if any expensive dtrace probes are enabled and we should go to the
9878 * debug loop.
9879 *
9880 * @returns true if we should use debug loop, false if not.
9881 */
9882static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9883{
9884 /* It's probably faster to OR the raw 32-bit counter variables together.
9885 Since the variables are in an array and the probes are next to one
9886 another (more or less), we have good locality. So, better read
9887 eight-nine cache lines ever time and only have one conditional, than
9888 128+ conditionals, right? */
9889 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9890 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9891 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9892 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9893 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9894 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9895 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9896 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9897 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9898 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9899 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9900 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9901 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9902 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9903 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9904 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9905 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9906 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9907 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9908 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9909 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9910 ) != 0
9911 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9912 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9913 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9914 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9915 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9916 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9917 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9918 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9919 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9920 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9921 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9922 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9923 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9924 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9925 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9926 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9927 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9928 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9929 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9930 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9932 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9935 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9936 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9937 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9938 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9939 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9940 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9941 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9942 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9943 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9956 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9957 ) != 0
9958 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9959 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9960 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9961 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9962 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9963 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9964 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9965 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9966 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9967 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9968 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9969 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9970 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9971 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9972 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9973 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9974 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9975 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9976 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9977 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9978 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9980 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9983 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9984 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9985 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9986 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9987 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9988 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9989 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9990 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9991 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10008 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10009 ) != 0;
10010}
10011
10012
10013/**
10014 * Runs the guest code using VT-x.
10015 *
10016 * @returns Strict VBox status code (i.e. informational status codes too).
10017 * @param pVCpu The cross context virtual CPU structure.
10018 */
10019VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10020{
10021 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10022 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10023 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10024 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10025
10026 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10027
10028 VBOXSTRICTRC rcStrict;
10029 if ( !pVCpu->hm.s.fUseDebugLoop
10030 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10031 && !DBGFIsStepping(pVCpu)
10032 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10033 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10034 else
10035 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10036
10037 if (rcStrict == VERR_EM_INTERPRETER)
10038 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10039 else if (rcStrict == VINF_EM_RESET)
10040 rcStrict = VINF_EM_TRIPLE_FAULT;
10041
10042 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10043 if (RT_FAILURE(rc2))
10044 {
10045 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10046 rcStrict = rc2;
10047 }
10048 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10049 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10050 return rcStrict;
10051}
10052
10053
10054#ifndef HMVMX_USE_FUNCTION_TABLE
10055DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10056{
10057#ifdef DEBUG_ramshankar
10058#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10059 do { \
10060 if (a_fSave != 0) \
10061 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10062 VBOXSTRICTRC rcStrict = a_CallExpr; \
10063 if (a_fSave != 0) \
10064 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10065 return rcStrict; \
10066 } while (0)
10067#else
10068# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10069#endif
10070 switch (rcReason)
10071 {
10072 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10073 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10074 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10075 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10076 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10077 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10078 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10079 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10080 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10081 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10082 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10083 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10084 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10085 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10086 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10087 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10088 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10089 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10090 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10091 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10092 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10093 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10094 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10095 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10096 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10097 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10098 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10099 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10100 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10101 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10102 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10103 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10104 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10105 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10106
10107 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10108 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10109 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10110 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10111 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10112 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10113 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10114 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10115 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10116
10117 case VMX_EXIT_VMCLEAR:
10118 case VMX_EXIT_VMLAUNCH:
10119 case VMX_EXIT_VMPTRLD:
10120 case VMX_EXIT_VMPTRST:
10121 case VMX_EXIT_VMREAD:
10122 case VMX_EXIT_VMRESUME:
10123 case VMX_EXIT_VMWRITE:
10124 case VMX_EXIT_VMXOFF:
10125 case VMX_EXIT_VMXON:
10126 case VMX_EXIT_INVEPT:
10127 case VMX_EXIT_INVVPID:
10128 case VMX_EXIT_VMFUNC:
10129 case VMX_EXIT_XSAVES:
10130 case VMX_EXIT_XRSTORS:
10131 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10132
10133 case VMX_EXIT_ENCLS:
10134 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10135 case VMX_EXIT_PML_FULL:
10136 default:
10137 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10138 }
10139#undef VMEXIT_CALL_RET
10140}
10141#endif /* !HMVMX_USE_FUNCTION_TABLE */
10142
10143
10144#ifdef VBOX_STRICT
10145/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10146# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10147 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10148
10149# define HMVMX_ASSERT_PREEMPT_CPUID() \
10150 do { \
10151 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10152 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10153 } while (0)
10154
10155# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10156 do { \
10157 AssertPtr((a_pVCpu)); \
10158 AssertPtr((a_pVmxTransient)); \
10159 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10160 Assert(ASMIntAreEnabled()); \
10161 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10162 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10163 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10164 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10165 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10166 HMVMX_ASSERT_PREEMPT_CPUID(); \
10167 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10168 } while (0)
10169
10170# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10171 do { \
10172 Log4Func(("\n")); \
10173 } while (0)
10174#else
10175# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10176 do { \
10177 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10178 RT_NOREF2((a_pVCpu), (a_pVmxTransient)); \
10179 } while (0)
10180# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10181#endif
10182
10183
10184/**
10185 * Advances the guest RIP by the specified number of bytes.
10186 *
10187 * @param pVCpu The cross context virtual CPU structure.
10188 * @param cbInstr Number of bytes to advance the RIP by.
10189 *
10190 * @remarks No-long-jump zone!!!
10191 */
10192DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10193{
10194 /* Advance the RIP. */
10195 pVCpu->cpum.GstCtx.rip += cbInstr;
10196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10197
10198 /* Update interrupt inhibition. */
10199 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10200 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10202}
10203
10204
10205/**
10206 * Advances the guest RIP after reading it from the VMCS.
10207 *
10208 * @returns VBox status code, no informational status codes.
10209 * @param pVCpu The cross context virtual CPU structure.
10210 * @param pVmxTransient Pointer to the VMX transient structure.
10211 *
10212 * @remarks No-long-jump zone!!!
10213 */
10214static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10215{
10216 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10217 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10218 AssertRCReturn(rc, rc);
10219
10220 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10221
10222 /*
10223 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10224 * pending debug exception field as it takes care of priority of events.
10225 *
10226 * See Intel spec. 32.2.1 "Debug Exceptions".
10227 */
10228 if ( !pVCpu->hm.s.fSingleInstruction
10229 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10230 {
10231 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10232 AssertRCReturn(rc, rc);
10233 }
10234
10235 return VINF_SUCCESS;
10236}
10237
10238
10239/**
10240 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10241 * and update error record fields accordingly.
10242 *
10243 * @return VMX_IGS_* return codes.
10244 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10245 * wrong with the guest state.
10246 *
10247 * @param pVCpu The cross context virtual CPU structure.
10248 *
10249 * @remarks This function assumes our cache of the VMCS controls
10250 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10251 */
10252static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10253{
10254#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10255#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10256 uError = (err); \
10257 break; \
10258 } else do { } while (0)
10259
10260 int rc;
10261 PVM pVM = pVCpu->CTX_SUFF(pVM);
10262 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10263 uint32_t uError = VMX_IGS_ERROR;
10264 uint32_t u32Val;
10265 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10266
10267 do
10268 {
10269 /*
10270 * CR0.
10271 */
10272 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10273 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10274 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10275 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10276 if (fUnrestrictedGuest)
10277 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10278
10279 uint32_t u32GuestCr0;
10280 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10281 AssertRCBreak(rc);
10282 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10283 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10284 if ( !fUnrestrictedGuest
10285 && (u32GuestCr0 & X86_CR0_PG)
10286 && !(u32GuestCr0 & X86_CR0_PE))
10287 {
10288 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10289 }
10290
10291 /*
10292 * CR4.
10293 */
10294 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10295 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10296
10297 uint32_t u32GuestCr4;
10298 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10299 AssertRCBreak(rc);
10300 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10301 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10302
10303 /*
10304 * IA32_DEBUGCTL MSR.
10305 */
10306 uint64_t u64Val;
10307 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10308 AssertRCBreak(rc);
10309 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10310 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10311 {
10312 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10313 }
10314 uint64_t u64DebugCtlMsr = u64Val;
10315
10316#ifdef VBOX_STRICT
10317 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10318 AssertRCBreak(rc);
10319 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10320#endif
10321 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10322
10323 /*
10324 * RIP and RFLAGS.
10325 */
10326 uint32_t u32Eflags;
10327#if HC_ARCH_BITS == 64
10328 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10329 AssertRCBreak(rc);
10330 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10331 if ( !fLongModeGuest
10332 || !pCtx->cs.Attr.n.u1Long)
10333 {
10334 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10335 }
10336 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10337 * must be identical if the "IA-32e mode guest" VM-entry
10338 * control is 1 and CS.L is 1. No check applies if the
10339 * CPU supports 64 linear-address bits. */
10340
10341 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10342 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10343 AssertRCBreak(rc);
10344 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10345 VMX_IGS_RFLAGS_RESERVED);
10346 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10347 u32Eflags = u64Val;
10348#else
10349 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10350 AssertRCBreak(rc);
10351 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10352 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10353#endif
10354
10355 if ( fLongModeGuest
10356 || ( fUnrestrictedGuest
10357 && !(u32GuestCr0 & X86_CR0_PE)))
10358 {
10359 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10360 }
10361
10362 uint32_t u32EntryInfo;
10363 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10364 AssertRCBreak(rc);
10365 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10366 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10367 {
10368 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10369 }
10370
10371 /*
10372 * 64-bit checks.
10373 */
10374#if HC_ARCH_BITS == 64
10375 if (fLongModeGuest)
10376 {
10377 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10378 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10379 }
10380
10381 if ( !fLongModeGuest
10382 && (u32GuestCr4 & X86_CR4_PCIDE))
10383 {
10384 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10385 }
10386
10387 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10388 * 51:32 beyond the processor's physical-address width are 0. */
10389
10390 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10391 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10392 {
10393 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10394 }
10395
10396 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10397 AssertRCBreak(rc);
10398 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10399
10400 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10401 AssertRCBreak(rc);
10402 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10403#endif
10404
10405 /*
10406 * PERF_GLOBAL MSR.
10407 */
10408 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10409 {
10410 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10411 AssertRCBreak(rc);
10412 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10413 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10414 }
10415
10416 /*
10417 * PAT MSR.
10418 */
10419 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10420 {
10421 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10422 AssertRCBreak(rc);
10423 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10424 for (unsigned i = 0; i < 8; i++)
10425 {
10426 uint8_t u8Val = (u64Val & 0xff);
10427 if ( u8Val != 0 /* UC */
10428 && u8Val != 1 /* WC */
10429 && u8Val != 4 /* WT */
10430 && u8Val != 5 /* WP */
10431 && u8Val != 6 /* WB */
10432 && u8Val != 7 /* UC- */)
10433 {
10434 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10435 }
10436 u64Val >>= 8;
10437 }
10438 }
10439
10440 /*
10441 * EFER MSR.
10442 */
10443 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10444 {
10445 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10446 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10447 AssertRCBreak(rc);
10448 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10449 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10450 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10451 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10452 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10453 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10454 || !(u32GuestCr0 & X86_CR0_PG)
10455 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10456 VMX_IGS_EFER_LMA_LME_MISMATCH);
10457 }
10458
10459 /*
10460 * Segment registers.
10461 */
10462 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10463 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10464 if (!(u32Eflags & X86_EFL_VM))
10465 {
10466 /* CS */
10467 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10468 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10469 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10470 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10471 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10472 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10473 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10474 /* CS cannot be loaded with NULL in protected mode. */
10475 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10476 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10477 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10479 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10480 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10481 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10482 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10483 else
10484 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10485
10486 /* SS */
10487 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10488 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10489 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10490 if ( !(pCtx->cr0 & X86_CR0_PE)
10491 || pCtx->cs.Attr.n.u4Type == 3)
10492 {
10493 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10494 }
10495 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10496 {
10497 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10498 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10499 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10500 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10501 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10502 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10503 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10504 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10505 }
10506
10507 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10508 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10509 {
10510 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10511 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10512 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10513 || pCtx->ds.Attr.n.u4Type > 11
10514 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10515 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10516 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10517 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10518 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10519 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10520 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10521 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10522 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10523 }
10524 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10525 {
10526 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10527 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10528 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10529 || pCtx->es.Attr.n.u4Type > 11
10530 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10531 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10532 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10533 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10534 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10535 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10536 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10537 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10538 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10539 }
10540 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10541 {
10542 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10543 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10544 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10545 || pCtx->fs.Attr.n.u4Type > 11
10546 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10547 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10548 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10549 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10550 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10551 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10552 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10553 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10554 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10555 }
10556 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10557 {
10558 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10559 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10560 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10561 || pCtx->gs.Attr.n.u4Type > 11
10562 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10563 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10564 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10565 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10566 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10567 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10568 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10569 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10570 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10571 }
10572 /* 64-bit capable CPUs. */
10573#if HC_ARCH_BITS == 64
10574 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10575 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10576 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10577 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10578 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10579 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10580 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10581 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10582 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10583 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10584 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10585#endif
10586 }
10587 else
10588 {
10589 /* V86 mode checks. */
10590 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10591 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10592 {
10593 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10594 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10595 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10596 }
10597 else
10598 {
10599 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10600 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10601 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10602 }
10603
10604 /* CS */
10605 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10606 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10607 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10608 /* SS */
10609 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10610 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10611 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10612 /* DS */
10613 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10614 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10615 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10616 /* ES */
10617 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10618 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10619 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10620 /* FS */
10621 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10622 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10623 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10624 /* GS */
10625 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10626 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10627 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10628 /* 64-bit capable CPUs. */
10629#if HC_ARCH_BITS == 64
10630 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10631 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10632 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10633 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10634 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10635 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10636 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10637 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10638 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10639 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10640 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10641#endif
10642 }
10643
10644 /*
10645 * TR.
10646 */
10647 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10648 /* 64-bit capable CPUs. */
10649#if HC_ARCH_BITS == 64
10650 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10651#endif
10652 if (fLongModeGuest)
10653 {
10654 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10655 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10656 }
10657 else
10658 {
10659 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10660 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10661 VMX_IGS_TR_ATTR_TYPE_INVALID);
10662 }
10663 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10664 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10665 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10666 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10667 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10668 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10669 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10670 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10671
10672 /*
10673 * GDTR and IDTR.
10674 */
10675#if HC_ARCH_BITS == 64
10676 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10677 AssertRCBreak(rc);
10678 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10679
10680 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10681 AssertRCBreak(rc);
10682 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10683#endif
10684
10685 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10686 AssertRCBreak(rc);
10687 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10688
10689 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10690 AssertRCBreak(rc);
10691 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10692
10693 /*
10694 * Guest Non-Register State.
10695 */
10696 /* Activity State. */
10697 uint32_t u32ActivityState;
10698 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10699 AssertRCBreak(rc);
10700 HMVMX_CHECK_BREAK( !u32ActivityState
10701 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10702 VMX_IGS_ACTIVITY_STATE_INVALID);
10703 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10704 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10705 uint32_t u32IntrState;
10706 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10707 AssertRCBreak(rc);
10708 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10709 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10710 {
10711 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10712 }
10713
10714 /** @todo Activity state and injecting interrupts. Left as a todo since we
10715 * currently don't use activity states but ACTIVE. */
10716
10717 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10718 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10719
10720 /* Guest interruptibility-state. */
10721 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10722 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10723 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10724 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10725 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10726 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10727 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10728 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10729 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10730 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10731 {
10732 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10733 {
10734 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10735 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10736 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10737 }
10738 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10739 {
10740 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10741 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10742 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10743 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10744 }
10745 }
10746 /** @todo Assumes the processor is not in SMM. */
10747 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10748 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10749 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10750 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10751 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10752 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10753 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10754 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10755 {
10756 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10757 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10758 }
10759
10760 /* Pending debug exceptions. */
10761#if HC_ARCH_BITS == 64
10762 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10763 AssertRCBreak(rc);
10764 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10765 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10766 u32Val = u64Val; /* For pending debug exceptions checks below. */
10767#else
10768 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10769 AssertRCBreak(rc);
10770 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10771 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10772#endif
10773
10774 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10775 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10776 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10777 {
10778 if ( (u32Eflags & X86_EFL_TF)
10779 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10780 {
10781 /* Bit 14 is PendingDebug.BS. */
10782 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10783 }
10784 if ( !(u32Eflags & X86_EFL_TF)
10785 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10786 {
10787 /* Bit 14 is PendingDebug.BS. */
10788 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10789 }
10790 }
10791
10792 /* VMCS link pointer. */
10793 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10794 AssertRCBreak(rc);
10795 if (u64Val != UINT64_C(0xffffffffffffffff))
10796 {
10797 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10798 /** @todo Bits beyond the processor's physical-address width MBZ. */
10799 /** @todo 32-bit located in memory referenced by value of this field (as a
10800 * physical address) must contain the processor's VMCS revision ID. */
10801 /** @todo SMM checks. */
10802 }
10803
10804 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10805 * not using Nested Paging? */
10806 if ( pVM->hm.s.fNestedPaging
10807 && !fLongModeGuest
10808 && CPUMIsGuestInPAEModeEx(pCtx))
10809 {
10810 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10811 AssertRCBreak(rc);
10812 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10813
10814 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10815 AssertRCBreak(rc);
10816 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10817
10818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10821
10822 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10823 AssertRCBreak(rc);
10824 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10825 }
10826
10827 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10828 if (uError == VMX_IGS_ERROR)
10829 uError = VMX_IGS_REASON_NOT_FOUND;
10830 } while (0);
10831
10832 pVCpu->hm.s.u32HMError = uError;
10833 return uError;
10834
10835#undef HMVMX_ERROR_BREAK
10836#undef HMVMX_CHECK_BREAK
10837}
10838
10839/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10840/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10841/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10842
10843/** @name VM-exit handlers.
10844 * @{
10845 */
10846
10847/**
10848 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10849 */
10850HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10851{
10852 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10854 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10855 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10856 return VINF_SUCCESS;
10857 return VINF_EM_RAW_INTERRUPT;
10858}
10859
10860
10861/**
10862 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10863 */
10864HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10865{
10866 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10867 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10868
10869 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10870 AssertRCReturn(rc, rc);
10871
10872 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10873 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10874 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10875 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10876
10877 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10878 {
10879 /*
10880 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10881 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10882 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10883 *
10884 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10885 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10886 */
10887 VMXDispatchHostNmi();
10888 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10889 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10890 return VINF_SUCCESS;
10891 }
10892
10893 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10894 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10895 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10896 { /* likely */ }
10897 else
10898 {
10899 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10900 rcStrictRc1 = VINF_SUCCESS;
10901 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10902 return rcStrictRc1;
10903 }
10904
10905 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10906 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10907 switch (uIntType)
10908 {
10909 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10910 Assert(uVector == X86_XCPT_DB);
10911 RT_FALL_THRU();
10912 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10913 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10914 RT_FALL_THRU();
10915 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10916 {
10917 /*
10918 * If there's any exception caused as a result of event injection, the resulting
10919 * secondary/final execption will be pending, we shall continue guest execution
10920 * after injecting the event. The page-fault case is complicated and we manually
10921 * handle any currently pending event in hmR0VmxExitXcptPF.
10922 */
10923 if (!pVCpu->hm.s.Event.fPending)
10924 { /* likely */ }
10925 else if (uVector != X86_XCPT_PF)
10926 {
10927 rc = VINF_SUCCESS;
10928 break;
10929 }
10930
10931 switch (uVector)
10932 {
10933 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10934 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10935 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10936 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10937 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10938 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10939
10940 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10941 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10942 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10943 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10944 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10945 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10946 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10947 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10948 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10949 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10950 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10951 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10952 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10953 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10954 default:
10955 {
10956 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10957 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10958 {
10959 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10960 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10961 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10962
10963 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10964 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10965 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10966 AssertRCReturn(rc, rc);
10967 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10968 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10969 0 /* GCPtrFaultAddress */);
10970 }
10971 else
10972 {
10973 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10974 pVCpu->hm.s.u32HMError = uVector;
10975 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10976 }
10977 break;
10978 }
10979 }
10980 break;
10981 }
10982
10983 default:
10984 {
10985 pVCpu->hm.s.u32HMError = uExitIntInfo;
10986 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10987 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10988 break;
10989 }
10990 }
10991 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10992 return rc;
10993}
10994
10995
10996/**
10997 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10998 */
10999HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11000{
11001 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11002
11003 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11004 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11005
11006 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11008 return VINF_SUCCESS;
11009}
11010
11011
11012/**
11013 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11014 */
11015HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11016{
11017 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11018 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11019 {
11020 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11021 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11022 }
11023
11024 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11025
11026 /*
11027 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11028 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11029 */
11030 uint32_t fIntrState = 0;
11031 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11032 AssertRCReturn(rc, rc);
11033
11034 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11035 if ( fBlockSti
11036 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11037 {
11038 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11039 }
11040
11041 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11042 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11043
11044 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11045 return VINF_SUCCESS;
11046}
11047
11048
11049/**
11050 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11051 */
11052HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11053{
11054 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11055 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11056}
11057
11058
11059/**
11060 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11061 */
11062HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11063{
11064 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11065 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11066}
11067
11068
11069/**
11070 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11071 */
11072HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11073{
11074 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11075
11076 /*
11077 * Get the state we need and update the exit history entry.
11078 */
11079 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11080 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11081 AssertRCReturn(rc, rc);
11082
11083 VBOXSTRICTRC rcStrict;
11084 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11085 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11086 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11087 if (!pExitRec)
11088 {
11089 /*
11090 * Regular CPUID instruction execution.
11091 */
11092 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11093 if (rcStrict == VINF_SUCCESS)
11094 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11095 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11096 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11097 {
11098 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11099 rcStrict = VINF_SUCCESS;
11100 }
11101 }
11102 else
11103 {
11104 /*
11105 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11106 */
11107 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11108 AssertRCReturn(rc2, rc2);
11109
11110 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11111 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11112
11113 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11114 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11115
11116 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11117 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11118 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11119 }
11120 return rcStrict;
11121}
11122
11123
11124/**
11125 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11126 */
11127HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11128{
11129 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11130 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11131 AssertRCReturn(rc, rc);
11132
11133 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11134 return VINF_EM_RAW_EMULATE_INSTR;
11135
11136 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11137 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11138}
11139
11140
11141/**
11142 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11143 */
11144HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11145{
11146 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11147 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11148 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11149 AssertRCReturn(rc, rc);
11150
11151 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11152 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11153 {
11154 /* If we get a spurious VM-exit when offsetting is enabled,
11155 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11156 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11157 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11158 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11159 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11160 }
11161 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11162 {
11163 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11164 rcStrict = VINF_SUCCESS;
11165 }
11166 return rcStrict;
11167}
11168
11169
11170/**
11171 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11172 */
11173HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11174{
11175 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11176 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11177 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11178 AssertRCReturn(rc, rc);
11179
11180 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11181 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11182 {
11183 /* If we get a spurious VM-exit when offsetting is enabled,
11184 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11185 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11186 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11187 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11188 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11189 }
11190 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11191 {
11192 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11193 rcStrict = VINF_SUCCESS;
11194 }
11195 return rcStrict;
11196}
11197
11198
11199/**
11200 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11201 */
11202HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11203{
11204 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11205 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11206 AssertRCReturn(rc, rc);
11207
11208 PVM pVM = pVCpu->CTX_SUFF(pVM);
11209 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11210 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11211 if (RT_LIKELY(rc == VINF_SUCCESS))
11212 {
11213 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11214 Assert(pVmxTransient->cbInstr == 2);
11215 }
11216 else
11217 {
11218 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11219 rc = VERR_EM_INTERPRETER;
11220 }
11221 return rc;
11222}
11223
11224
11225/**
11226 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11227 */
11228HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11229{
11230 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11231
11232 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11233 if (EMAreHypercallInstructionsEnabled(pVCpu))
11234 {
11235 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11236 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11237 AssertRCReturn(rc, rc);
11238
11239 /* Perform the hypercall. */
11240 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11241 if (rcStrict == VINF_SUCCESS)
11242 {
11243 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11244 AssertRCReturn(rc, rc);
11245 }
11246 else
11247 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11248 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11249 || RT_FAILURE(rcStrict));
11250
11251 /* If the hypercall changes anything other than guest's general-purpose registers,
11252 we would need to reload the guest changed bits here before VM-entry. */
11253 }
11254 else
11255 Log4Func(("Hypercalls not enabled\n"));
11256
11257 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11258 if (RT_FAILURE(rcStrict))
11259 {
11260 hmR0VmxSetPendingXcptUD(pVCpu);
11261 rcStrict = VINF_SUCCESS;
11262 }
11263
11264 return rcStrict;
11265}
11266
11267
11268/**
11269 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11270 */
11271HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11272{
11273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11274 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11275
11276 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11277 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11278 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11279 AssertRCReturn(rc, rc);
11280
11281 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11282
11283 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11284 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11285 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11286 {
11287 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11288 rcStrict = VINF_SUCCESS;
11289 }
11290 else
11291 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11292 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11293 return rcStrict;
11294}
11295
11296
11297/**
11298 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11299 */
11300HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11301{
11302 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11303 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11304 AssertRCReturn(rc, rc);
11305
11306 PVM pVM = pVCpu->CTX_SUFF(pVM);
11307 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11308 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11309 if (RT_LIKELY(rc == VINF_SUCCESS))
11310 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11311 else
11312 {
11313 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11314 rc = VERR_EM_INTERPRETER;
11315 }
11316 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11317 return rc;
11318}
11319
11320
11321/**
11322 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11323 */
11324HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11325{
11326 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11327 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11328 AssertRCReturn(rc, rc);
11329
11330 PVM pVM = pVCpu->CTX_SUFF(pVM);
11331 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11332 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11333 rc = VBOXSTRICTRC_VAL(rc2);
11334 if (RT_LIKELY( rc == VINF_SUCCESS
11335 || rc == VINF_EM_HALT))
11336 {
11337 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11338 AssertRCReturn(rc3, rc3);
11339
11340 if ( rc == VINF_EM_HALT
11341 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11342 rc = VINF_SUCCESS;
11343 }
11344 else
11345 {
11346 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11347 rc = VERR_EM_INTERPRETER;
11348 }
11349 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11350 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11351 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11352 return rc;
11353}
11354
11355
11356/**
11357 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11358 */
11359HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11360{
11361 /*
11362 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11363 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11364 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11365 * VMX root operation. If we get here, something funny is going on.
11366 *
11367 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11368 */
11369 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11370 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11371 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11372}
11373
11374
11375/**
11376 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11377 */
11378HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11379{
11380 /*
11381 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11382 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11383 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11384 * an SMI. If we get here, something funny is going on.
11385 *
11386 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11387 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11388 */
11389 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11390 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11391 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11392}
11393
11394
11395/**
11396 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11397 */
11398HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11399{
11400 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11401 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11402 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11403 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11404}
11405
11406
11407/**
11408 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11409 */
11410HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11411{
11412 /*
11413 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11414 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11415 * See Intel spec. 25.3 "Other Causes of VM-exits".
11416 */
11417 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11418 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11419 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11420}
11421
11422
11423/**
11424 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11425 * VM-exit.
11426 */
11427HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11428{
11429 /*
11430 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11431 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11432 *
11433 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11434 * See Intel spec. "23.8 Restrictions on VMX operation".
11435 */
11436 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11437 return VINF_SUCCESS;
11438}
11439
11440
11441/**
11442 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11443 * VM-exit.
11444 */
11445HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11446{
11447 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11448 return VINF_EM_RESET;
11449}
11450
11451
11452/**
11453 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11454 */
11455HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11456{
11457 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11458 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11459
11460 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11461 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11462 AssertRCReturn(rc, rc);
11463
11464 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11465 rc = VINF_SUCCESS;
11466 else
11467 rc = VINF_EM_HALT;
11468
11469 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11470 if (rc != VINF_SUCCESS)
11471 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11472 return rc;
11473}
11474
11475
11476/**
11477 * VM-exit handler for instructions that result in a \#UD exception delivered to
11478 * the guest.
11479 */
11480HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11481{
11482 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11483 hmR0VmxSetPendingXcptUD(pVCpu);
11484 return VINF_SUCCESS;
11485}
11486
11487
11488/**
11489 * VM-exit handler for expiry of the VMX preemption timer.
11490 */
11491HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11492{
11493 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11494
11495 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11496 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11497
11498 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11499 PVM pVM = pVCpu->CTX_SUFF(pVM);
11500 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11502 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11503}
11504
11505
11506/**
11507 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11508 */
11509HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11510{
11511 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11512
11513 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11514 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11515 AssertRCReturn(rc, rc);
11516
11517 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11518 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11519 : HM_CHANGED_XCPT_RAISED_MASK);
11520
11521 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11522 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11523
11524 return rcStrict;
11525}
11526
11527
11528/**
11529 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11530 */
11531HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11532{
11533 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11534 /** @todo Use VM-exit instruction information. */
11535 return VERR_EM_INTERPRETER;
11536}
11537
11538
11539/**
11540 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11541 * Error VM-exit.
11542 */
11543HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11544{
11545 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11546 AssertRCReturn(rc, rc);
11547 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11548 if (RT_FAILURE(rc))
11549 return rc;
11550
11551 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11552 NOREF(uInvalidReason);
11553
11554#ifdef VBOX_STRICT
11555 uint32_t fIntrState;
11556 RTHCUINTREG uHCReg;
11557 uint64_t u64Val;
11558 uint32_t u32Val;
11559
11560 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11561 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11562 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11563 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11564 AssertRCReturn(rc, rc);
11565
11566 Log4(("uInvalidReason %u\n", uInvalidReason));
11567 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11568 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11569 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11570 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11571
11572 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11573 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11574 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11575 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11576 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11577 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11578 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11579 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11580 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11581 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11582 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11583 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11584
11585 hmR0DumpRegs(pVCpu);
11586#else
11587 NOREF(pVmxTransient);
11588#endif
11589
11590 return VERR_VMX_INVALID_GUEST_STATE;
11591}
11592
11593
11594/**
11595 * VM-exit handler for VM-entry failure due to an MSR-load
11596 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11597 */
11598HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11599{
11600 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11601 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11602}
11603
11604
11605/**
11606 * VM-exit handler for VM-entry failure due to a machine-check event
11607 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11608 */
11609HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11610{
11611 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11612 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11613}
11614
11615
11616/**
11617 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11618 * theory.
11619 */
11620HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11621{
11622 RT_NOREF2(pVCpu, pVmxTransient);
11623 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11624 return VERR_VMX_UNDEFINED_EXIT_CODE;
11625}
11626
11627
11628/**
11629 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11630 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11631 * Conditional VM-exit.
11632 */
11633HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11634{
11635 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11636
11637 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11638 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11639 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11640 return VERR_EM_INTERPRETER;
11641 AssertMsgFailed(("Unexpected XDTR access\n"));
11642 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11643}
11644
11645
11646/**
11647 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11648 */
11649HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11650{
11651 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11652
11653 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11654 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11655 return VERR_EM_INTERPRETER;
11656 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11657 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11658}
11659
11660
11661/**
11662 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11663 */
11664HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11665{
11666 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11667
11668 /** @todo Optimize this: We currently drag in in the whole MSR state
11669 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11670 * MSRs required. That would require changes to IEM and possibly CPUM too.
11671 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11672 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11673 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11674 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11675 AssertRCReturn(rc, rc);
11676
11677 Log4Func(("ecx=%#RX32\n", idMsr));
11678
11679#ifdef VBOX_STRICT
11680 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11681 {
11682 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11683 && idMsr != MSR_K6_EFER)
11684 {
11685 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11686 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11687 }
11688 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11689 {
11690 VMXMSREXITREAD enmRead;
11691 VMXMSREXITWRITE enmWrite;
11692 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11693 AssertRCReturn(rc2, rc2);
11694 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11695 {
11696 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11697 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11698 }
11699 }
11700 }
11701#endif
11702
11703 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11704 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11705 if (rcStrict == VINF_SUCCESS)
11706 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11707 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11708 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11709 {
11710 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11711 rcStrict = VINF_SUCCESS;
11712 }
11713 else
11714 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11715
11716 return rcStrict;
11717}
11718
11719
11720/**
11721 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11722 */
11723HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11724{
11725 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11726
11727 /** @todo Optimize this: We currently drag in in the whole MSR state
11728 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11729 * MSRs required. That would require changes to IEM and possibly CPUM too.
11730 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11731 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11732 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11733 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11734 AssertRCReturn(rc, rc);
11735
11736 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11737
11738 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11739 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11740
11741 if (rcStrict == VINF_SUCCESS)
11742 {
11743 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11744
11745 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11746 if ( idMsr == MSR_IA32_APICBASE
11747 || ( idMsr >= MSR_IA32_X2APIC_START
11748 && idMsr <= MSR_IA32_X2APIC_END))
11749 {
11750 /*
11751 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11752 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11753 */
11754 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11755 }
11756 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11757 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11758 else if (idMsr == MSR_K6_EFER)
11759 {
11760 /*
11761 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11762 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11763 * the other bits as well, SCE and NXE. See @bugref{7368}.
11764 */
11765 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11766 | HM_CHANGED_VMX_EXIT_CTLS);
11767 }
11768
11769 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11770 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11771 {
11772 switch (idMsr)
11773 {
11774 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11775 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11776 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11777 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11778 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11779 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11780 default:
11781 {
11782 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11783 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11784 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11785 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11786 break;
11787 }
11788 }
11789 }
11790#ifdef VBOX_STRICT
11791 else
11792 {
11793 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11794 switch (idMsr)
11795 {
11796 case MSR_IA32_SYSENTER_CS:
11797 case MSR_IA32_SYSENTER_EIP:
11798 case MSR_IA32_SYSENTER_ESP:
11799 case MSR_K8_FS_BASE:
11800 case MSR_K8_GS_BASE:
11801 {
11802 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11803 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11804 }
11805
11806 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11807 default:
11808 {
11809 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11810 {
11811 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11812 if (idMsr != MSR_K6_EFER)
11813 {
11814 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11815 idMsr));
11816 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11817 }
11818 }
11819
11820 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11821 {
11822 VMXMSREXITREAD enmRead;
11823 VMXMSREXITWRITE enmWrite;
11824 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11825 AssertRCReturn(rc2, rc2);
11826 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11827 {
11828 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11829 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11830 }
11831 }
11832 break;
11833 }
11834 }
11835 }
11836#endif /* VBOX_STRICT */
11837 }
11838 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11839 {
11840 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11841 rcStrict = VINF_SUCCESS;
11842 }
11843 else
11844 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11845
11846 return rcStrict;
11847}
11848
11849
11850/**
11851 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11852 */
11853HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11854{
11855 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11856 /** @todo The guest has likely hit a contended spinlock. We might want to
11857 * poke a schedule different guest VCPU. */
11858 return VINF_EM_RAW_INTERRUPT;
11859}
11860
11861
11862/**
11863 * VM-exit handler for when the TPR value is lowered below the specified
11864 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11865 */
11866HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11867{
11868 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11869 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11870
11871 /*
11872 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11873 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11874 */
11875 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11876 return VINF_SUCCESS;
11877}
11878
11879
11880/**
11881 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11882 * VM-exit.
11883 *
11884 * @retval VINF_SUCCESS when guest execution can continue.
11885 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11886 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11887 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11888 * interpreter.
11889 */
11890HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11891{
11892 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11893 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11894
11895 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11896 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11897 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11898 AssertRCReturn(rc, rc);
11899
11900 VBOXSTRICTRC rcStrict;
11901 PVM pVM = pVCpu->CTX_SUFF(pVM);
11902 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11903 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11904 switch (uAccessType)
11905 {
11906 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11907 {
11908 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11909 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11910 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11911 AssertMsg( rcStrict == VINF_SUCCESS
11912 || rcStrict == VINF_IEM_RAISED_XCPT
11913 || rcStrict == VINF_PGM_CHANGE_MODE
11914 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11915
11916 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11917 {
11918 case 0:
11919 {
11920 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11921 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11922 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11923 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11924 break;
11925 }
11926
11927 case 2:
11928 {
11929 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11930 /* Nothing to do here, CR2 it's not part of the VMCS. */
11931 break;
11932 }
11933
11934 case 3:
11935 {
11936 Assert( !pVM->hm.s.fNestedPaging
11937 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11938 || pVCpu->hm.s.fUsingDebugLoop);
11939 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11940 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11941 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11942 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11943 break;
11944 }
11945
11946 case 4:
11947 {
11948 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11949 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11950 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11951 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11952 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11953 break;
11954 }
11955
11956 case 8:
11957 {
11958 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11959 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11960 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11961 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11962 break;
11963 }
11964 default:
11965 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11966 break;
11967 }
11968 break;
11969 }
11970
11971 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11972 {
11973 Assert( !pVM->hm.s.fNestedPaging
11974 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11975 || pVCpu->hm.s.fUsingDebugLoop
11976 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11977 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11978 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11979 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11980
11981 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11982 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11983 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11984 AssertMsg( rcStrict == VINF_SUCCESS
11985 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11986#ifdef VBOX_WITH_STATISTICS
11987 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11988 {
11989 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11990 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11991 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11992 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11993 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11994 }
11995#endif
11996 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11997 VBOXSTRICTRC_VAL(rcStrict)));
11998 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
11999 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12000 else
12001 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12002 break;
12003 }
12004
12005 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12006 {
12007 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12008 AssertMsg( rcStrict == VINF_SUCCESS
12009 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12010
12011 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12013 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12014 break;
12015 }
12016
12017 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12018 {
12019 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12020 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12021 AssertMsg( rcStrict == VINF_SUCCESS
12022 || rcStrict == VINF_IEM_RAISED_XCPT
12023 || rcStrict == VINF_PGM_CHANGE_MODE,
12024 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12025
12026 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12027 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12028 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12029 break;
12030 }
12031
12032 default:
12033 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12034 VERR_VMX_UNEXPECTED_EXCEPTION);
12035 }
12036
12037 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12038 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12039 if (rcStrict == VINF_IEM_RAISED_XCPT)
12040 {
12041 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
12042 rcStrict = VINF_SUCCESS;
12043 }
12044
12045 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12046 NOREF(pVM);
12047 return rcStrict;
12048}
12049
12050
12051/**
12052 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12053 * VM-exit.
12054 */
12055HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12056{
12057 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12058 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12059
12060 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12061 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12062 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12063 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12064 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12065 AssertRCReturn(rc, rc);
12066
12067 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12068 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12069 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12070 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12071 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12072 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12073 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12074 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12075 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12076
12077 /*
12078 * Update exit history to see if this exit can be optimized.
12079 */
12080 VBOXSTRICTRC rcStrict;
12081 PCEMEXITREC pExitRec = NULL;
12082 if ( !fGstStepping
12083 && !fDbgStepping)
12084 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12085 !fIOString
12086 ? !fIOWrite
12087 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12088 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12089 : !fIOWrite
12090 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12091 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12092 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12093 if (!pExitRec)
12094 {
12095 /* I/O operation lookup arrays. */
12096 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12097 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12098 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12099 uint32_t const cbInstr = pVmxTransient->cbInstr;
12100 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12101 PVM pVM = pVCpu->CTX_SUFF(pVM);
12102 if (fIOString)
12103 {
12104 /*
12105 * INS/OUTS - I/O String instruction.
12106 *
12107 * Use instruction-information if available, otherwise fall back on
12108 * interpreting the instruction.
12109 */
12110 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12111 fIOWrite ? 'w' : 'r'));
12112 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12113 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12114 {
12115 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12116 AssertRCReturn(rc2, rc2);
12117 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12118 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12119 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12120 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12121 if (fIOWrite)
12122 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12123 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12124 else
12125 {
12126 /*
12127 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12128 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12129 * See Intel Instruction spec. for "INS".
12130 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12131 */
12132 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12133 }
12134 }
12135 else
12136 rcStrict = IEMExecOne(pVCpu);
12137
12138 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12139 fUpdateRipAlready = true;
12140 }
12141 else
12142 {
12143 /*
12144 * IN/OUT - I/O instruction.
12145 */
12146 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12147 fIOWrite ? 'w' : 'r'));
12148 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12149 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12150 if (fIOWrite)
12151 {
12152 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12154 }
12155 else
12156 {
12157 uint32_t u32Result = 0;
12158 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12159 if (IOM_SUCCESS(rcStrict))
12160 {
12161 /* Save result of I/O IN instr. in AL/AX/EAX. */
12162 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12163 }
12164 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12165 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12167 }
12168 }
12169
12170 if (IOM_SUCCESS(rcStrict))
12171 {
12172 if (!fUpdateRipAlready)
12173 {
12174 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12175 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12176 }
12177
12178 /*
12179 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12180 * while booting Fedora 17 64-bit guest.
12181 *
12182 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12183 */
12184 if (fIOString)
12185 {
12186 /** @todo Single-step for INS/OUTS with REP prefix? */
12187 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12188 }
12189 else if ( !fDbgStepping
12190 && fGstStepping)
12191 {
12192 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12193 AssertRCReturn(rc, rc);
12194 }
12195
12196 /*
12197 * If any I/O breakpoints are armed, we need to check if one triggered
12198 * and take appropriate action.
12199 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12200 */
12201 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12202 AssertRCReturn(rc, rc);
12203
12204 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12205 * execution engines about whether hyper BPs and such are pending. */
12206 uint32_t const uDr7 = pCtx->dr[7];
12207 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12208 && X86_DR7_ANY_RW_IO(uDr7)
12209 && (pCtx->cr4 & X86_CR4_DE))
12210 || DBGFBpIsHwIoArmed(pVM)))
12211 {
12212 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12213
12214 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12215 VMMRZCallRing3Disable(pVCpu);
12216 HM_DISABLE_PREEMPT(pVCpu);
12217
12218 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12219
12220 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12221 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12222 {
12223 /* Raise #DB. */
12224 if (fIsGuestDbgActive)
12225 ASMSetDR6(pCtx->dr[6]);
12226 if (pCtx->dr[7] != uDr7)
12227 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12228
12229 hmR0VmxSetPendingXcptDB(pVCpu);
12230 }
12231 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12232 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12233 else if ( rcStrict2 != VINF_SUCCESS
12234 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12235 rcStrict = rcStrict2;
12236 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12237
12238 HM_RESTORE_PREEMPT();
12239 VMMRZCallRing3Enable(pVCpu);
12240 }
12241 }
12242
12243#ifdef VBOX_STRICT
12244 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12245 Assert(!fIOWrite);
12246 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12247 Assert(fIOWrite);
12248 else
12249 {
12250# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12251 * statuses, that the VMM device and some others may return. See
12252 * IOM_SUCCESS() for guidance. */
12253 AssertMsg( RT_FAILURE(rcStrict)
12254 || rcStrict == VINF_SUCCESS
12255 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12256 || rcStrict == VINF_EM_DBG_BREAKPOINT
12257 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12258 || rcStrict == VINF_EM_RAW_TO_R3
12259 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12260# endif
12261 }
12262#endif
12263 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12264 }
12265 else
12266 {
12267 /*
12268 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12269 */
12270 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12271 AssertRCReturn(rc2, rc2);
12272 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12273 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12274 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12275 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12276 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12277 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12278
12279 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12280 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12281
12282 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12283 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12284 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12285 }
12286 return rcStrict;
12287}
12288
12289
12290/**
12291 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12292 * VM-exit.
12293 */
12294HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12295{
12296 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12297
12298 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12299 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12300 AssertRCReturn(rc, rc);
12301 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12302 {
12303 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12304 AssertRCReturn(rc, rc);
12305 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12306 {
12307 uint32_t uErrCode;
12308 RTGCUINTPTR GCPtrFaultAddress;
12309 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12310 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12311 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12312 if (fErrorCodeValid)
12313 {
12314 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12315 AssertRCReturn(rc, rc);
12316 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12317 }
12318 else
12319 uErrCode = 0;
12320
12321 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12322 && uVector == X86_XCPT_PF)
12323 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12324 else
12325 GCPtrFaultAddress = 0;
12326
12327 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12328 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12329
12330 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12331 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12332 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12333 }
12334 }
12335
12336 /* Fall back to the interpreter to emulate the task-switch. */
12337 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12338 return VERR_EM_INTERPRETER;
12339}
12340
12341
12342/**
12343 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12344 */
12345HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12346{
12347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12348 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12349 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12350 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12351 AssertRCReturn(rc, rc);
12352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12353 return VINF_EM_DBG_STEPPED;
12354}
12355
12356
12357/**
12358 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12359 */
12360HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12361{
12362 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12363
12364 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12365
12366 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12367 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12368 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12369 {
12370 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12371 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12372 {
12373 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12374 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12375 }
12376 }
12377 else
12378 {
12379 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12380 rcStrict1 = VINF_SUCCESS;
12381 return rcStrict1;
12382 }
12383
12384 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12385 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12386 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12387 AssertRCReturn(rc, rc);
12388
12389 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12390 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12391 VBOXSTRICTRC rcStrict2;
12392 switch (uAccessType)
12393 {
12394 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12395 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12396 {
12397 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12398 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12399 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12400
12401 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12402 GCPhys &= PAGE_BASE_GC_MASK;
12403 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12404 PVM pVM = pVCpu->CTX_SUFF(pVM);
12405 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12406 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12407
12408 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12409 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12410 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12411 CPUMCTX2CORE(pCtx), GCPhys);
12412 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12413 if ( rcStrict2 == VINF_SUCCESS
12414 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12415 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12416 {
12417 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12418 | HM_CHANGED_GUEST_APIC_TPR);
12419 rcStrict2 = VINF_SUCCESS;
12420 }
12421 break;
12422 }
12423
12424 default:
12425 Log4Func(("uAccessType=%#x\n", uAccessType));
12426 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12427 break;
12428 }
12429
12430 if (rcStrict2 != VINF_SUCCESS)
12431 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12432 return rcStrict2;
12433}
12434
12435
12436/**
12437 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12438 * VM-exit.
12439 */
12440HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12441{
12442 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12443
12444 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12445 if (pVmxTransient->fWasGuestDebugStateActive)
12446 {
12447 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12448 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12449 }
12450
12451 if ( !pVCpu->hm.s.fSingleInstruction
12452 && !pVmxTransient->fWasHyperDebugStateActive)
12453 {
12454 Assert(!DBGFIsStepping(pVCpu));
12455 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12456
12457 /* Don't intercept MOV DRx any more. */
12458 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12459 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12460 AssertRCReturn(rc, rc);
12461
12462 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12463 VMMRZCallRing3Disable(pVCpu);
12464 HM_DISABLE_PREEMPT(pVCpu);
12465
12466 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12467 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12468 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12469
12470 HM_RESTORE_PREEMPT();
12471 VMMRZCallRing3Enable(pVCpu);
12472
12473#ifdef VBOX_WITH_STATISTICS
12474 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12475 AssertRCReturn(rc, rc);
12476 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12477 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12478 else
12479 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12480#endif
12481 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12482 return VINF_SUCCESS;
12483 }
12484
12485 /*
12486 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12487 * Update the segment registers and DR7 from the CPU.
12488 */
12489 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12490 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12491 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12492 AssertRCReturn(rc, rc);
12493 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12494
12495 PVM pVM = pVCpu->CTX_SUFF(pVM);
12496 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12497 {
12498 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12499 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12500 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12501 if (RT_SUCCESS(rc))
12502 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12504 }
12505 else
12506 {
12507 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12508 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12509 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12510 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12511 }
12512
12513 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12514 if (RT_SUCCESS(rc))
12515 {
12516 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12517 AssertRCReturn(rc2, rc2);
12518 return VINF_SUCCESS;
12519 }
12520 return rc;
12521}
12522
12523
12524/**
12525 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12526 * Conditional VM-exit.
12527 */
12528HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12529{
12530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12531 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12532
12533 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12534 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12535 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12536 {
12537 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12538 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12539 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12540 {
12541 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12542 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12543 }
12544 }
12545 else
12546 {
12547 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12548 rcStrict1 = VINF_SUCCESS;
12549 return rcStrict1;
12550 }
12551
12552 /*
12553 * Get sufficent state and update the exit history entry.
12554 */
12555 RTGCPHYS GCPhys;
12556 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12557 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12558 AssertRCReturn(rc, rc);
12559
12560 VBOXSTRICTRC rcStrict;
12561 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12562 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12563 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12564 if (!pExitRec)
12565 {
12566 /*
12567 * If we succeed, resume guest execution.
12568 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12569 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12570 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12571 * weird case. See @bugref{6043}.
12572 */
12573 PVM pVM = pVCpu->CTX_SUFF(pVM);
12574 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12575 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12576 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12577 if ( rcStrict == VINF_SUCCESS
12578 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12579 || rcStrict == VERR_PAGE_NOT_PRESENT)
12580 {
12581 /* Successfully handled MMIO operation. */
12582 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12583 | HM_CHANGED_GUEST_APIC_TPR);
12584 rcStrict = VINF_SUCCESS;
12585 }
12586 }
12587 else
12588 {
12589 /*
12590 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12591 */
12592 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12593 AssertRCReturn(rc2, rc2);
12594
12595 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12596 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12597
12598 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12599 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12600
12601 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12602 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12603 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12604 }
12605 return VBOXSTRICTRC_TODO(rcStrict);
12606}
12607
12608
12609/**
12610 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12611 * VM-exit.
12612 */
12613HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12614{
12615 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12616 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12617
12618 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12619 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12620 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12621 {
12622 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12623 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12624 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12625 }
12626 else
12627 {
12628 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12629 rcStrict1 = VINF_SUCCESS;
12630 return rcStrict1;
12631 }
12632
12633 RTGCPHYS GCPhys;
12634 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12635 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12636 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12637 AssertRCReturn(rc, rc);
12638
12639 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12640 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12641
12642 RTGCUINT uErrorCode = 0;
12643 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12644 uErrorCode |= X86_TRAP_PF_ID;
12645 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12646 uErrorCode |= X86_TRAP_PF_RW;
12647 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12648 uErrorCode |= X86_TRAP_PF_P;
12649
12650 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12651
12652
12653 /* Handle the pagefault trap for the nested shadow table. */
12654 PVM pVM = pVCpu->CTX_SUFF(pVM);
12655 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12656
12657 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12658 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12659
12660 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12661 TRPMResetTrap(pVCpu);
12662
12663 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12664 if ( rcStrict2 == VINF_SUCCESS
12665 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12666 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12667 {
12668 /* Successfully synced our nested page tables. */
12669 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12670 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12671 return VINF_SUCCESS;
12672 }
12673
12674 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12675 return rcStrict2;
12676}
12677
12678/** @} */
12679
12680/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12681/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12682/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12683
12684/** @name VM-exit exception handlers.
12685 * @{
12686 */
12687
12688/**
12689 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12690 */
12691static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12692{
12693 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12694 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12695
12696 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12697 AssertRCReturn(rc, rc);
12698
12699 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12700 {
12701 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12702 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12703
12704 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12705 * provides VM-exit instruction length. If this causes problem later,
12706 * disassemble the instruction like it's done on AMD-V. */
12707 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12708 AssertRCReturn(rc2, rc2);
12709 return rc;
12710 }
12711
12712 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12713 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12714 return rc;
12715}
12716
12717
12718/**
12719 * VM-exit exception handler for \#BP (Breakpoint exception).
12720 */
12721static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12722{
12723 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12724 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12725
12726 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12727 AssertRCReturn(rc, rc);
12728
12729 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12730 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12731 if (rc == VINF_EM_RAW_GUEST_TRAP)
12732 {
12733 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12734 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12735 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12736 AssertRCReturn(rc, rc);
12737
12738 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12739 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12740 }
12741
12742 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12743 return rc;
12744}
12745
12746
12747/**
12748 * VM-exit exception handler for \#AC (alignment check exception).
12749 */
12750static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12751{
12752 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12753
12754 /*
12755 * Re-inject it. We'll detect any nesting before getting here.
12756 */
12757 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12758 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12759 AssertRCReturn(rc, rc);
12760 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12761
12762 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12763 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12764 return VINF_SUCCESS;
12765}
12766
12767
12768/**
12769 * VM-exit exception handler for \#DB (Debug exception).
12770 */
12771static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12772{
12773 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12774 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12775
12776 /*
12777 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12778 * for processing.
12779 */
12780 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12781
12782 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12783 uint64_t uDR6 = X86_DR6_INIT_VAL;
12784 uDR6 |= ( pVmxTransient->uExitQualification
12785 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12786
12787 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12788 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12789 Log6Func(("rc=%Rrc\n", rc));
12790 if (rc == VINF_EM_RAW_GUEST_TRAP)
12791 {
12792 /*
12793 * The exception was for the guest. Update DR6, DR7.GD and
12794 * IA32_DEBUGCTL.LBR before forwarding it.
12795 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12796 */
12797 VMMRZCallRing3Disable(pVCpu);
12798 HM_DISABLE_PREEMPT(pVCpu);
12799
12800 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12801 pCtx->dr[6] |= uDR6;
12802 if (CPUMIsGuestDebugStateActive(pVCpu))
12803 ASMSetDR6(pCtx->dr[6]);
12804
12805 HM_RESTORE_PREEMPT();
12806 VMMRZCallRing3Enable(pVCpu);
12807
12808 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12809 AssertRCReturn(rc, rc);
12810
12811 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12812 pCtx->dr[7] &= ~X86_DR7_GD;
12813
12814 /* Paranoia. */
12815 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12816 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12817
12818 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12819 AssertRCReturn(rc, rc);
12820
12821 /*
12822 * Raise #DB in the guest.
12823 *
12824 * It is important to reflect exactly what the VM-exit gave us (preserving the
12825 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12826 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12827 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12828 *
12829 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12830 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12831 */
12832 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12833 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12834 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12835 AssertRCReturn(rc, rc);
12836 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12837 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12838 return VINF_SUCCESS;
12839 }
12840
12841 /*
12842 * Not a guest trap, must be a hypervisor related debug event then.
12843 * Update DR6 in case someone is interested in it.
12844 */
12845 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12846 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12847 CPUMSetHyperDR6(pVCpu, uDR6);
12848
12849 return rc;
12850}
12851
12852/**
12853 * VM-exit exception handler for \#GP (General-protection exception).
12854 *
12855 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12856 */
12857static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12858{
12859 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12860 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12861
12862 int rc;
12863 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12864 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12865 { /* likely */ }
12866 else
12867 {
12868#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12869 Assert(pVCpu->hm.s.fUsingDebugLoop);
12870#endif
12871 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12872 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12873 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12874 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12875 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12876 AssertRCReturn(rc, rc);
12877 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12878 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12879 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12880 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12881 return rc;
12882 }
12883
12884 Assert(CPUMIsGuestInRealModeEx(pCtx));
12885 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12886
12887 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12888 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12889 AssertRCReturn(rc, rc);
12890
12891 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12892 uint32_t cbOp = 0;
12893 PVM pVM = pVCpu->CTX_SUFF(pVM);
12894 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12895 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12896 if (RT_SUCCESS(rc))
12897 {
12898 rc = VINF_SUCCESS;
12899 Assert(cbOp == pDis->cbInstr);
12900 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12901 switch (pDis->pCurInstr->uOpcode)
12902 {
12903 case OP_CLI:
12904 {
12905 pCtx->eflags.Bits.u1IF = 0;
12906 pCtx->eflags.Bits.u1RF = 0;
12907 pCtx->rip += pDis->cbInstr;
12908 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12909 if ( !fDbgStepping
12910 && pCtx->eflags.Bits.u1TF)
12911 {
12912 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12913 AssertRCReturn(rc, rc);
12914 }
12915 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12916 break;
12917 }
12918
12919 case OP_STI:
12920 {
12921 bool fOldIF = pCtx->eflags.Bits.u1IF;
12922 pCtx->eflags.Bits.u1IF = 1;
12923 pCtx->eflags.Bits.u1RF = 0;
12924 pCtx->rip += pDis->cbInstr;
12925 if (!fOldIF)
12926 {
12927 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12928 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12929 }
12930 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12931 if ( !fDbgStepping
12932 && pCtx->eflags.Bits.u1TF)
12933 {
12934 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12935 AssertRCReturn(rc, rc);
12936 }
12937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12938 break;
12939 }
12940
12941 case OP_HLT:
12942 {
12943 rc = VINF_EM_HALT;
12944 pCtx->rip += pDis->cbInstr;
12945 pCtx->eflags.Bits.u1RF = 0;
12946 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12947 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12948 break;
12949 }
12950
12951 case OP_POPF:
12952 {
12953 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12954 uint32_t cbParm;
12955 uint32_t uMask;
12956 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12957 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12958 {
12959 cbParm = 4;
12960 uMask = 0xffffffff;
12961 }
12962 else
12963 {
12964 cbParm = 2;
12965 uMask = 0xffff;
12966 }
12967
12968 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12969 RTGCPTR GCPtrStack = 0;
12970 X86EFLAGS Eflags;
12971 Eflags.u32 = 0;
12972 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12973 &GCPtrStack);
12974 if (RT_SUCCESS(rc))
12975 {
12976 Assert(sizeof(Eflags.u32) >= cbParm);
12977 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12978 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12979 }
12980 if (RT_FAILURE(rc))
12981 {
12982 rc = VERR_EM_INTERPRETER;
12983 break;
12984 }
12985 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
12986 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
12987 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12988 pCtx->esp += cbParm;
12989 pCtx->esp &= uMask;
12990 pCtx->rip += pDis->cbInstr;
12991 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12992 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12993 POPF restores EFLAGS.TF. */
12994 if ( !fDbgStepping
12995 && fGstStepping)
12996 {
12997 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12998 AssertRCReturn(rc, rc);
12999 }
13000 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13001 break;
13002 }
13003
13004 case OP_PUSHF:
13005 {
13006 uint32_t cbParm;
13007 uint32_t uMask;
13008 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13009 {
13010 cbParm = 4;
13011 uMask = 0xffffffff;
13012 }
13013 else
13014 {
13015 cbParm = 2;
13016 uMask = 0xffff;
13017 }
13018
13019 /* Get the stack pointer & push the contents of eflags onto the stack. */
13020 RTGCPTR GCPtrStack = 0;
13021 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13022 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13023 if (RT_FAILURE(rc))
13024 {
13025 rc = VERR_EM_INTERPRETER;
13026 break;
13027 }
13028 X86EFLAGS Eflags = pCtx->eflags;
13029 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13030 Eflags.Bits.u1RF = 0;
13031 Eflags.Bits.u1VM = 0;
13032
13033 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13034 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13035 {
13036 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13037 rc = VERR_EM_INTERPRETER;
13038 break;
13039 }
13040 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13041 pCtx->esp -= cbParm;
13042 pCtx->esp &= uMask;
13043 pCtx->rip += pDis->cbInstr;
13044 pCtx->eflags.Bits.u1RF = 0;
13045 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13046 if ( !fDbgStepping
13047 && pCtx->eflags.Bits.u1TF)
13048 {
13049 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13050 AssertRCReturn(rc, rc);
13051 }
13052 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13053 break;
13054 }
13055
13056 case OP_IRET:
13057 {
13058 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13059 * instruction reference. */
13060 RTGCPTR GCPtrStack = 0;
13061 uint32_t uMask = 0xffff;
13062 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13063 uint16_t aIretFrame[3];
13064 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13065 {
13066 rc = VERR_EM_INTERPRETER;
13067 break;
13068 }
13069 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13070 &GCPtrStack);
13071 if (RT_SUCCESS(rc))
13072 {
13073 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13074 PGMACCESSORIGIN_HM));
13075 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13076 }
13077 if (RT_FAILURE(rc))
13078 {
13079 rc = VERR_EM_INTERPRETER;
13080 break;
13081 }
13082 pCtx->eip = 0;
13083 pCtx->ip = aIretFrame[0];
13084 pCtx->cs.Sel = aIretFrame[1];
13085 pCtx->cs.ValidSel = aIretFrame[1];
13086 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13087 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13088 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13089 pCtx->sp += sizeof(aIretFrame);
13090 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13091 | HM_CHANGED_GUEST_CS);
13092 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13093 if ( !fDbgStepping
13094 && fGstStepping)
13095 {
13096 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13097 AssertRCReturn(rc, rc);
13098 }
13099 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13100 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13101 break;
13102 }
13103
13104 case OP_INT:
13105 {
13106 uint16_t uVector = pDis->Param1.uValue & 0xff;
13107 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13108 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13109 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13110 break;
13111 }
13112
13113 case OP_INTO:
13114 {
13115 if (pCtx->eflags.Bits.u1OF)
13116 {
13117 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13118 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13120 }
13121 else
13122 {
13123 pCtx->eflags.Bits.u1RF = 0;
13124 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13125 }
13126 break;
13127 }
13128
13129 default:
13130 {
13131 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13132 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13133 EMCODETYPE_SUPERVISOR);
13134 rc = VBOXSTRICTRC_VAL(rc2);
13135 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13136 /** @todo We have to set pending-debug exceptions here when the guest is
13137 * single-stepping depending on the instruction that was interpreted. */
13138 Log4Func(("#GP rc=%Rrc\n", rc));
13139 break;
13140 }
13141 }
13142 }
13143 else
13144 rc = VERR_EM_INTERPRETER;
13145
13146 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13147 ("#GP Unexpected rc=%Rrc\n", rc));
13148 return rc;
13149}
13150
13151
13152/**
13153 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13154 * the exception reported in the VMX transient structure back into the VM.
13155 *
13156 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13157 * up-to-date.
13158 */
13159static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13160{
13161 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13162#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13163 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13164 ("uVector=%#x u32XcptBitmap=%#X32\n",
13165 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13166#endif
13167
13168 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13169 hmR0VmxCheckExitDueToEventDelivery(). */
13170 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13171 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13172 AssertRCReturn(rc, rc);
13173 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13174
13175#ifdef DEBUG_ramshankar
13176 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13177 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13178 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13179#endif
13180
13181 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13182 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13183 return VINF_SUCCESS;
13184}
13185
13186
13187/**
13188 * VM-exit exception handler for \#PF (Page-fault exception).
13189 */
13190static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13191{
13192 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13193 PVM pVM = pVCpu->CTX_SUFF(pVM);
13194 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13195 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13196 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13197 AssertRCReturn(rc, rc);
13198
13199 if (!pVM->hm.s.fNestedPaging)
13200 { /* likely */ }
13201 else
13202 {
13203#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13204 Assert(pVCpu->hm.s.fUsingDebugLoop);
13205#endif
13206 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13207 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13208 {
13209 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13210 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13211 }
13212 else
13213 {
13214 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13215 hmR0VmxSetPendingXcptDF(pVCpu);
13216 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13217 }
13218 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13219 return rc;
13220 }
13221
13222 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13223 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13224 if (pVmxTransient->fVectoringPF)
13225 {
13226 Assert(pVCpu->hm.s.Event.fPending);
13227 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13228 }
13229
13230 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13231 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13232 AssertRCReturn(rc, rc);
13233
13234 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13235 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13236
13237 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13238 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13239 (RTGCPTR)pVmxTransient->uExitQualification);
13240
13241 Log4Func(("#PF: rc=%Rrc\n", rc));
13242 if (rc == VINF_SUCCESS)
13243 {
13244 /*
13245 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13246 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13247 */
13248 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13249 TRPMResetTrap(pVCpu);
13250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13251 return rc;
13252 }
13253
13254 if (rc == VINF_EM_RAW_GUEST_TRAP)
13255 {
13256 if (!pVmxTransient->fVectoringDoublePF)
13257 {
13258 /* It's a guest page fault and needs to be reflected to the guest. */
13259 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13260 TRPMResetTrap(pVCpu);
13261 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13262 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13263 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13264 }
13265 else
13266 {
13267 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13268 TRPMResetTrap(pVCpu);
13269 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13270 hmR0VmxSetPendingXcptDF(pVCpu);
13271 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13272 }
13273
13274 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13275 return VINF_SUCCESS;
13276 }
13277
13278 TRPMResetTrap(pVCpu);
13279 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13280 return rc;
13281}
13282
13283/** @} */
13284
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette