VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72985

Last change on this file since 72985 was 72985, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: bugref:9193 Stop passing pCtx around and use pVCpu->cpum.GstCtx instead where possible. [build fix]
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 562.1 KB
Line 
1/* $Id: HMVMXR0.cpp 72985 2018-07-08 16:34:43Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
152 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001FFF)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1174 {
1175 iBit = uMsr - UINT32_C(0xC0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001FFF)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xC0000000
1219 && uMsr <= 0xC0001FFF)
1220 {
1221 iBit = (uMsr - 0xC0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1254 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1255 {
1256 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1257 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1259 }
1260
1261 /* Update number of guest MSRs to load/store across the world-switch. */
1262 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1263 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1264
1265 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1266 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1267 AssertRCReturn(rc, rc);
1268
1269 /* Update the VCPU's copy of the MSR count. */
1270 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1271
1272 return VINF_SUCCESS;
1273}
1274
1275
1276/**
1277 * Adds a new (or updates the value of an existing) guest/host MSR
1278 * pair to be swapped during the world-switch as part of the
1279 * auto-load/store MSR area in the VMCS.
1280 *
1281 * @returns VBox status code.
1282 * @param pVCpu The cross context virtual CPU structure.
1283 * @param uMsr The MSR.
1284 * @param uGuestMsrValue Value of the guest MSR.
1285 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1286 * necessary.
1287 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1288 * its value was updated. Optional, can be NULL.
1289 */
1290static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1291 bool *pfAddedAndUpdated)
1292{
1293 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1294 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1295 uint32_t i;
1296 for (i = 0; i < cMsrs; i++)
1297 {
1298 if (pGuestMsr->u32Msr == uMsr)
1299 break;
1300 pGuestMsr++;
1301 }
1302
1303 bool fAdded = false;
1304 if (i == cMsrs)
1305 {
1306 ++cMsrs;
1307 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1308 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1309
1310 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1311 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1312 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1313
1314 fAdded = true;
1315 }
1316
1317 /* Update the MSR values in the auto-load/store MSR area. */
1318 pGuestMsr->u32Msr = uMsr;
1319 pGuestMsr->u64Value = uGuestMsrValue;
1320
1321 /* Create/update the MSR slot in the host MSR area. */
1322 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1323 pHostMsr += i;
1324 pHostMsr->u32Msr = uMsr;
1325
1326 /*
1327 * Update the host MSR only when requested by the caller AND when we're
1328 * adding it to the auto-load/store area. Otherwise, it would have been
1329 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1330 */
1331 bool fUpdatedMsrValue = false;
1332 if ( fAdded
1333 && fUpdateHostMsr)
1334 {
1335 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1336 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1337 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1338 fUpdatedMsrValue = true;
1339 }
1340
1341 if (pfAddedAndUpdated)
1342 *pfAddedAndUpdated = fUpdatedMsrValue;
1343 return VINF_SUCCESS;
1344}
1345
1346
1347/**
1348 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1349 * auto-load/store MSR area in the VMCS.
1350 *
1351 * @returns VBox status code.
1352 * @param pVCpu The cross context virtual CPU structure.
1353 * @param uMsr The MSR.
1354 */
1355static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1356{
1357 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1358 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1359 for (uint32_t i = 0; i < cMsrs; i++)
1360 {
1361 /* Find the MSR. */
1362 if (pGuestMsr->u32Msr == uMsr)
1363 {
1364 /* If it's the last MSR, simply reduce the count. */
1365 if (i == cMsrs - 1)
1366 {
1367 --cMsrs;
1368 break;
1369 }
1370
1371 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1372 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1373 pLastGuestMsr += cMsrs - 1;
1374 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1375 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1376
1377 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1378 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 pLastHostMsr += cMsrs - 1;
1380 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1381 pHostMsr->u64Value = pLastHostMsr->u64Value;
1382 --cMsrs;
1383 break;
1384 }
1385 pGuestMsr++;
1386 }
1387
1388 /* Update the VMCS if the count changed (meaning the MSR was found). */
1389 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1390 {
1391 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1392 AssertRCReturn(rc, rc);
1393
1394 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1396 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1397
1398 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1399 return VINF_SUCCESS;
1400 }
1401
1402 return VERR_NOT_FOUND;
1403}
1404
1405
1406/**
1407 * Checks if the specified guest MSR is part of the auto-load/store area in
1408 * the VMCS.
1409 *
1410 * @returns true if found, false otherwise.
1411 * @param pVCpu The cross context virtual CPU structure.
1412 * @param uMsr The MSR to find.
1413 */
1414static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1415{
1416 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1417 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1418
1419 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1420 {
1421 if (pGuestMsr->u32Msr == uMsr)
1422 return true;
1423 }
1424 return false;
1425}
1426
1427
1428/**
1429 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1430 *
1431 * @param pVCpu The cross context virtual CPU structure.
1432 *
1433 * @remarks No-long-jump zone!!!
1434 */
1435static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1436{
1437 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1438 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1439 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1440 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1441
1442 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1443 {
1444 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1445
1446 /*
1447 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1448 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1449 */
1450 if (pHostMsr->u32Msr == MSR_K6_EFER)
1451 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1452 else
1453 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1454 }
1455
1456 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1457}
1458
1459
1460/**
1461 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1462 * perform lazy restoration of the host MSRs while leaving VT-x.
1463 *
1464 * @param pVCpu The cross context virtual CPU structure.
1465 *
1466 * @remarks No-long-jump zone!!!
1467 */
1468static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1469{
1470 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1471
1472 /*
1473 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1474 */
1475 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1476 {
1477 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1478#if HC_ARCH_BITS == 64
1479 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1480 {
1481 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1482 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1483 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1484 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1485 }
1486#endif
1487 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1488 }
1489}
1490
1491
1492/**
1493 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1494 * lazily while leaving VT-x.
1495 *
1496 * @returns true if it does, false otherwise.
1497 * @param pVCpu The cross context virtual CPU structure.
1498 * @param uMsr The MSR to check.
1499 */
1500static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1501{
1502 NOREF(pVCpu);
1503#if HC_ARCH_BITS == 64
1504 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1505 {
1506 switch (uMsr)
1507 {
1508 case MSR_K8_LSTAR:
1509 case MSR_K6_STAR:
1510 case MSR_K8_SF_MASK:
1511 case MSR_K8_KERNEL_GS_BASE:
1512 return true;
1513 }
1514 }
1515#else
1516 RT_NOREF(pVCpu, uMsr);
1517#endif
1518 return false;
1519}
1520
1521
1522/**
1523 * Loads a set of guests MSRs to allow read/passthru to the guest.
1524 *
1525 * The name of this function is slightly confusing. This function does NOT
1526 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1527 * common prefix for functions dealing with "lazy restoration" of the shared
1528 * MSRs.
1529 *
1530 * @param pVCpu The cross context virtual CPU structure.
1531 *
1532 * @remarks No-long-jump zone!!!
1533 */
1534static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1535{
1536 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1537 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1538
1539 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1540#if HC_ARCH_BITS == 64
1541 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1542 {
1543 /*
1544 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1545 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1546 * we can skip a few MSR writes.
1547 *
1548 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1549 * guest MSR values in the guest-CPU context might be different to what's currently
1550 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1551 * CPU, see @bugref{8728}.
1552 */
1553 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1554 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1555 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1556 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1557 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1558 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1559 {
1560#ifdef VBOX_STRICT
1561 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1562 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1563 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1564 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1565#endif
1566 }
1567 else
1568 {
1569 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1570 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1571 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1572 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1573 }
1574 }
1575#endif
1576 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1577}
1578
1579
1580/**
1581 * Performs lazy restoration of the set of host MSRs if they were previously
1582 * loaded with guest MSR values.
1583 *
1584 * @param pVCpu The cross context virtual CPU structure.
1585 *
1586 * @remarks No-long-jump zone!!!
1587 * @remarks The guest MSRs should have been saved back into the guest-CPU
1588 * context by hmR0VmxImportGuestState()!!!
1589 */
1590static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1591{
1592 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1593 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1594
1595 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1596 {
1597 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1598#if HC_ARCH_BITS == 64
1599 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1600 {
1601 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1602 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1603 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1604 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1605 }
1606#endif
1607 }
1608 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1609}
1610
1611
1612/**
1613 * Verifies that our cached values of the VMCS fields are all consistent with
1614 * what's actually present in the VMCS.
1615 *
1616 * @returns VBox status code.
1617 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1618 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1619 * VMCS content. HMCPU error-field is
1620 * updated, see VMX_VCI_XXX.
1621 * @param pVCpu The cross context virtual CPU structure.
1622 */
1623static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1624{
1625 uint32_t u32Val;
1626 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1627 AssertRCReturn(rc, rc);
1628 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1629 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1630 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1631 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1632
1633 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1636 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1637 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1638 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1643 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1645 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1650 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1652 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1653
1654 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1655 {
1656 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1657 AssertRCReturn(rc, rc);
1658 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1659 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1660 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1661 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1662 }
1663
1664 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1665 AssertRCReturn(rc, rc);
1666 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1667 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1668 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1669 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1670
1671 uint64_t u64Val;
1672 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1673 AssertRCReturn(rc, rc);
1674 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1675 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1676 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1677 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1678
1679 return VINF_SUCCESS;
1680}
1681
1682
1683#ifdef VBOX_STRICT
1684/**
1685 * Verifies that our cached host EFER value has not changed
1686 * since we cached it.
1687 *
1688 * @param pVCpu The cross context virtual CPU structure.
1689 */
1690static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1691{
1692 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1693
1694 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1695 {
1696 uint64_t u64Val;
1697 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1698 AssertRC(rc);
1699
1700 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1701 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1702 }
1703}
1704
1705
1706/**
1707 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1708 * VMCS are correct.
1709 *
1710 * @param pVCpu The cross context virtual CPU structure.
1711 */
1712static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1713{
1714 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1715
1716 /* Verify MSR counts in the VMCS are what we think it should be. */
1717 uint32_t cMsrs;
1718 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1719 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1720
1721 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1722 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1723
1724 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1725 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1726
1727 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1728 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1729 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1730 {
1731 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1732 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1733 pGuestMsr->u32Msr, cMsrs));
1734
1735 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1736 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1737 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1738
1739 /* Verify that the permissions are as expected in the MSR bitmap. */
1740 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1741 {
1742 VMXMSREXITREAD enmRead;
1743 VMXMSREXITWRITE enmWrite;
1744 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1745 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1746 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1747 {
1748 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1749 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1750 }
1751 else
1752 {
1753 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1754 pGuestMsr->u32Msr, cMsrs));
1755 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1756 pGuestMsr->u32Msr, cMsrs));
1757 }
1758 }
1759 }
1760}
1761#endif /* VBOX_STRICT */
1762
1763
1764/**
1765 * Flushes the TLB using EPT.
1766 *
1767 * @returns VBox status code.
1768 * @param pVCpu The cross context virtual CPU structure of the calling
1769 * EMT. Can be NULL depending on @a enmTlbFlush.
1770 * @param enmTlbFlush Type of flush.
1771 *
1772 * @remarks Caller is responsible for making sure this function is called only
1773 * when NestedPaging is supported and providing @a enmTlbFlush that is
1774 * supported by the CPU.
1775 * @remarks Can be called with interrupts disabled.
1776 */
1777static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1778{
1779 uint64_t au64Descriptor[2];
1780 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1781 au64Descriptor[0] = 0;
1782 else
1783 {
1784 Assert(pVCpu);
1785 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1786 }
1787 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1788
1789 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1790 AssertMsg(rc == VINF_SUCCESS,
1791 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1792
1793 if ( RT_SUCCESS(rc)
1794 && pVCpu)
1795 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1796}
1797
1798
1799/**
1800 * Flushes the TLB using VPID.
1801 *
1802 * @returns VBox status code.
1803 * @param pVCpu The cross context virtual CPU structure of the calling
1804 * EMT. Can be NULL depending on @a enmTlbFlush.
1805 * @param enmTlbFlush Type of flush.
1806 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1807 * on @a enmTlbFlush).
1808 *
1809 * @remarks Can be called with interrupts disabled.
1810 */
1811static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1812{
1813 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1814
1815 uint64_t au64Descriptor[2];
1816 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1817 {
1818 au64Descriptor[0] = 0;
1819 au64Descriptor[1] = 0;
1820 }
1821 else
1822 {
1823 AssertPtr(pVCpu);
1824 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1827 au64Descriptor[1] = GCPtr;
1828 }
1829
1830 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1831 AssertMsg(rc == VINF_SUCCESS,
1832 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1833
1834 if ( RT_SUCCESS(rc)
1835 && pVCpu)
1836 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1837 NOREF(rc);
1838}
1839
1840
1841/**
1842 * Invalidates a guest page by guest virtual address. Only relevant for
1843 * EPT/VPID, otherwise there is nothing really to invalidate.
1844 *
1845 * @returns VBox status code.
1846 * @param pVCpu The cross context virtual CPU structure.
1847 * @param GCVirt Guest virtual address of the page to invalidate.
1848 */
1849VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1850{
1851 AssertPtr(pVCpu);
1852 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1853
1854 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1855 if (!fFlushPending)
1856 {
1857 /*
1858 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1859 * the EPT case. See @bugref{6043} and @bugref{6177}.
1860 *
1861 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1862 * as this function maybe called in a loop with individual addresses.
1863 */
1864 PVM pVM = pVCpu->CTX_SUFF(pVM);
1865 if (pVM->hm.s.vmx.fVpid)
1866 {
1867 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1868
1869#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1870 /*
1871 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1872 * where executing INVVPID outside 64-bit mode does not flush translations of
1873 * 64-bit linear addresses, see @bugref{6208#c72}.
1874 */
1875 if (RT_HI_U32(GCVirt))
1876 fVpidFlush = false;
1877#endif
1878
1879 if (fVpidFlush)
1880 {
1881 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1882 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1883 }
1884 else
1885 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1886 }
1887 else if (pVM->hm.s.fNestedPaging)
1888 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1889 }
1890
1891 return VINF_SUCCESS;
1892}
1893
1894
1895/**
1896 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1897 * case where neither EPT nor VPID is supported by the CPU.
1898 *
1899 * @param pVCpu The cross context virtual CPU structure.
1900 * @param pCpu Pointer to the global HM struct.
1901 *
1902 * @remarks Called with interrupts disabled.
1903 */
1904static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1905{
1906 AssertPtr(pVCpu);
1907 AssertPtr(pCpu);
1908
1909 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1910
1911 Assert(pCpu->idCpu != NIL_RTCPUID);
1912 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1913 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1914 pVCpu->hm.s.fForceTLBFlush = false;
1915 return;
1916}
1917
1918
1919/**
1920 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1921 *
1922 * @param pVCpu The cross context virtual CPU structure.
1923 * @param pCpu Pointer to the global HM CPU struct.
1924 *
1925 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1926 * nomenclature. The reason is, to avoid confusion in compare statements
1927 * since the host-CPU copies are named "ASID".
1928 *
1929 * @remarks Called with interrupts disabled.
1930 */
1931static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1932{
1933#ifdef VBOX_WITH_STATISTICS
1934 bool fTlbFlushed = false;
1935# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1936# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1937 if (!fTlbFlushed) \
1938 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1939 } while (0)
1940#else
1941# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1942# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1943#endif
1944
1945 AssertPtr(pCpu);
1946 AssertPtr(pVCpu);
1947 Assert(pCpu->idCpu != NIL_RTCPUID);
1948
1949 PVM pVM = pVCpu->CTX_SUFF(pVM);
1950 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1951 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1952 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1953
1954 /*
1955 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1956 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1957 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1958 * cannot reuse the current ASID anymore.
1959 */
1960 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1961 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1962 {
1963 ++pCpu->uCurrentAsid;
1964 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1965 {
1966 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1967 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1968 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1969 }
1970
1971 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1972 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1973 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1974
1975 /*
1976 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1977 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1978 */
1979 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1980 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1981 HMVMX_SET_TAGGED_TLB_FLUSHED();
1982 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1983 }
1984
1985 /* Check for explicit TLB flushes. */
1986 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1987 {
1988 /*
1989 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1990 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1991 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1992 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1993 * mappings, see @bugref{6568}.
1994 *
1995 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1996 */
1997 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1998 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1999 HMVMX_SET_TAGGED_TLB_FLUSHED();
2000 }
2001
2002 pVCpu->hm.s.fForceTLBFlush = false;
2003 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2004
2005 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2006 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2007 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2008 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2009 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2010 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2011 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2012 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2013 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2014
2015 /* Update VMCS with the VPID. */
2016 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2017 AssertRC(rc);
2018
2019#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2020}
2021
2022
2023/**
2024 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2025 *
2026 * @returns VBox status code.
2027 * @param pVCpu The cross context virtual CPU structure.
2028 * @param pCpu Pointer to the global HM CPU struct.
2029 *
2030 * @remarks Called with interrupts disabled.
2031 */
2032static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2033{
2034 AssertPtr(pVCpu);
2035 AssertPtr(pCpu);
2036 Assert(pCpu->idCpu != NIL_RTCPUID);
2037 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2038 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2039
2040 /*
2041 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2042 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2043 */
2044 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2045 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2046 {
2047 pVCpu->hm.s.fForceTLBFlush = true;
2048 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2049 }
2050
2051 /* Check for explicit TLB flushes. */
2052 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2053 {
2054 pVCpu->hm.s.fForceTLBFlush = true;
2055 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2056 }
2057
2058 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2059 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2060
2061 if (pVCpu->hm.s.fForceTLBFlush)
2062 {
2063 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2064 pVCpu->hm.s.fForceTLBFlush = false;
2065 }
2066}
2067
2068
2069/**
2070 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2071 *
2072 * @returns VBox status code.
2073 * @param pVCpu The cross context virtual CPU structure.
2074 * @param pCpu Pointer to the global HM CPU struct.
2075 *
2076 * @remarks Called with interrupts disabled.
2077 */
2078static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2079{
2080 AssertPtr(pVCpu);
2081 AssertPtr(pCpu);
2082 Assert(pCpu->idCpu != NIL_RTCPUID);
2083 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2084 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2085
2086 /*
2087 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2088 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2089 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2090 * cannot reuse the current ASID anymore.
2091 */
2092 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2093 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2094 {
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2097 }
2098
2099 /* Check for explicit TLB flushes. */
2100 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2101 {
2102 /*
2103 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2104 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2105 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2106 * include fExplicitFlush's too) - an obscure corner case.
2107 */
2108 pVCpu->hm.s.fForceTLBFlush = true;
2109 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2110 }
2111
2112 PVM pVM = pVCpu->CTX_SUFF(pVM);
2113 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2114 if (pVCpu->hm.s.fForceTLBFlush)
2115 {
2116 ++pCpu->uCurrentAsid;
2117 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2118 {
2119 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2120 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2121 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2122 }
2123
2124 pVCpu->hm.s.fForceTLBFlush = false;
2125 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2126 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2127 if (pCpu->fFlushAsidBeforeUse)
2128 {
2129 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2130 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2131 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2132 {
2133 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2134 pCpu->fFlushAsidBeforeUse = false;
2135 }
2136 else
2137 {
2138 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2139 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2140 }
2141 }
2142 }
2143
2144 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2145 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2146 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2147 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2148 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2149 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2150 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2151
2152 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2153 AssertRC(rc);
2154}
2155
2156
2157/**
2158 * Flushes the guest TLB entry based on CPU capabilities.
2159 *
2160 * @param pVCpu The cross context virtual CPU structure.
2161 * @param pCpu Pointer to the global HM CPU struct.
2162 */
2163DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2164{
2165#ifdef HMVMX_ALWAYS_FLUSH_TLB
2166 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2167#endif
2168 PVM pVM = pVCpu->CTX_SUFF(pVM);
2169 switch (pVM->hm.s.vmx.enmTlbFlushType)
2170 {
2171 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2174 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2175 default:
2176 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2177 break;
2178 }
2179 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2180}
2181
2182
2183/**
2184 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2185 * TLB entries from the host TLB before VM-entry.
2186 *
2187 * @returns VBox status code.
2188 * @param pVM The cross context VM structure.
2189 */
2190static int hmR0VmxSetupTaggedTlb(PVM pVM)
2191{
2192 /*
2193 * Determine optimal flush type for Nested Paging.
2194 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2195 * guest execution (see hmR3InitFinalizeR0()).
2196 */
2197 if (pVM->hm.s.fNestedPaging)
2198 {
2199 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2200 {
2201 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2202 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2203 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2204 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2205 else
2206 {
2207 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2208 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2209 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2210 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211 }
2212
2213 /* Make sure the write-back cacheable memory type for EPT is supported. */
2214 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2215 {
2216 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2217 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219 }
2220
2221 /* EPT requires a page-walk length of 4. */
2222 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2223 {
2224 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2225 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2226 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2227 }
2228 }
2229 else
2230 {
2231 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2232 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2233 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2234 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2235 }
2236 }
2237
2238 /*
2239 * Determine optimal flush type for VPID.
2240 */
2241 if (pVM->hm.s.vmx.fVpid)
2242 {
2243 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2244 {
2245 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2246 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2247 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2248 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2249 else
2250 {
2251 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2252 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2253 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2254 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2255 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2256 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2257 pVM->hm.s.vmx.fVpid = false;
2258 }
2259 }
2260 else
2261 {
2262 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2263 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2264 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2265 pVM->hm.s.vmx.fVpid = false;
2266 }
2267 }
2268
2269 /*
2270 * Setup the handler for flushing tagged-TLBs.
2271 */
2272 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2273 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2274 else if (pVM->hm.s.fNestedPaging)
2275 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2276 else if (pVM->hm.s.vmx.fVpid)
2277 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2278 else
2279 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2280 return VINF_SUCCESS;
2281}
2282
2283
2284/**
2285 * Sets up pin-based VM-execution controls in the VMCS.
2286 *
2287 * @returns VBox status code.
2288 * @param pVCpu The cross context virtual CPU structure.
2289 *
2290 * @remarks We don't really care about optimizing vmwrites here as it's done only
2291 * once per VM and hence we don't care about VMCS-field cache comparisons.
2292 */
2293static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2294{
2295 PVM pVM = pVCpu->CTX_SUFF(pVM);
2296 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2297 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2298
2299 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2300 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2301
2302 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2303 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2304
2305 /* Enable the VMX preemption timer. */
2306 if (pVM->hm.s.vmx.fUsePreemptTimer)
2307 {
2308 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2309 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2310 }
2311
2312#if 0
2313 /* Enable posted-interrupt processing. */
2314 if (pVM->hm.s.fPostedIntrs)
2315 {
2316 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2317 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2318 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2319 }
2320#endif
2321
2322 if ((fVal & fZap) != fVal)
2323 {
2324 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2325 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2326 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2327 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2328 }
2329
2330 /* Commit it to the VMCS and update our cache. */
2331 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2332 AssertRCReturn(rc, rc);
2333 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2334
2335 return VINF_SUCCESS;
2336}
2337
2338
2339/**
2340 * Sets up secondary processor-based VM-execution controls in the VMCS.
2341 *
2342 * @returns VBox status code.
2343 * @param pVCpu The cross context virtual CPU structure.
2344 *
2345 * @remarks We don't really care about optimizing vmwrites here as it's done only
2346 * once per VM and hence we don't care about VMCS-field cache comparisons.
2347 */
2348static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2349{
2350 PVM pVM = pVCpu->CTX_SUFF(pVM);
2351 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2352 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2353
2354 /* WBINVD causes a VM-exit. */
2355 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2357
2358 /* Enable EPT (aka nested-paging). */
2359 if (pVM->hm.s.fNestedPaging)
2360 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2361
2362 /*
2363 * Enable the INVPCID instruction if supported by the hardware and we expose
2364 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2365 */
2366 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2367 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2369
2370 /* Enable VPID. */
2371 if (pVM->hm.s.vmx.fVpid)
2372 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2373
2374 /* Enable Unrestricted guest execution. */
2375 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2376 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2377
2378#if 0
2379 if (pVM->hm.s.fVirtApicRegs)
2380 {
2381 /* Enable APIC-register virtualization. */
2382 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2384
2385 /* Enable virtual-interrupt delivery. */
2386 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2387 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2388 }
2389#endif
2390
2391 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2392 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2393 * done dynamically. */
2394 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2395 {
2396 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2397 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2398 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2399 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2400 AssertRCReturn(rc, rc);
2401 }
2402
2403 /* Enable RDTSCP. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2405 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2406
2407 /* Enable Pause-Loop exiting. */
2408 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2409 && pVM->hm.s.vmx.cPleGapTicks
2410 && pVM->hm.s.vmx.cPleWindowTicks)
2411 {
2412 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2413
2414 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2415 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2416 AssertRCReturn(rc, rc);
2417 }
2418
2419 if ((fVal & fZap) != fVal)
2420 {
2421 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2422 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2423 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2424 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2425 }
2426
2427 /* Commit it to the VMCS and update our cache. */
2428 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2429 AssertRCReturn(rc, rc);
2430 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2431
2432 return VINF_SUCCESS;
2433}
2434
2435
2436/**
2437 * Sets up processor-based VM-execution controls in the VMCS.
2438 *
2439 * @returns VBox status code.
2440 * @param pVCpu The cross context virtual CPU structure.
2441 *
2442 * @remarks We don't really care about optimizing vmwrites here as it's done only
2443 * once per VM and hence we don't care about VMCS-field cache comparisons.
2444 */
2445static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2446{
2447 PVM pVM = pVCpu->CTX_SUFF(pVM);
2448 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2449 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2450
2451 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2452 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2453 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2454 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2455 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2456 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2457 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2458
2459 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2460 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2461 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2462 {
2463 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2464 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2465 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2466 }
2467
2468 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2469 if (!pVM->hm.s.fNestedPaging)
2470 {
2471 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2473 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2474 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2475 }
2476
2477 /* Use TPR shadowing if supported by the CPU. */
2478 if ( PDMHasApic(pVM)
2479 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2480 {
2481 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2482 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2483 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2484 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2485 AssertRCReturn(rc, rc);
2486
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2488 /* CR8 writes cause a VM-exit based on TPR threshold. */
2489 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2490 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2491 }
2492 else
2493 {
2494 /*
2495 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2496 * Set this control only for 64-bit guests.
2497 */
2498 if (pVM->hm.s.fAllow64BitGuests)
2499 {
2500 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2501 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2502 }
2503 }
2504
2505 /* Use MSR-bitmaps if supported by the CPU. */
2506 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2507 {
2508 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2509
2510 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2511 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2512 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2513 AssertRCReturn(rc, rc);
2514
2515 /*
2516 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2517 * automatically using dedicated fields in the VMCS.
2518 */
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2524#if HC_ARCH_BITS == 64
2525 /*
2526 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2527 */
2528 if (pVM->hm.s.fAllow64BitGuests)
2529 {
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534 }
2535#endif
2536 /*
2537 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2538 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2539 */
2540 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2541 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542
2543 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2544 }
2545
2546 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2547 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2549
2550 if ((fVal & fZap) != fVal)
2551 {
2552 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2553 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Commit it to the VMCS and update our cache. */
2559 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2560 AssertRCReturn(rc, rc);
2561 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2562
2563 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2564 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2565 return hmR0VmxSetupProcCtls2(pVCpu);
2566
2567 /* Sanity check, should not really happen. */
2568 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2569 {
2570 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2571 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2572 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2573 }
2574
2575 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2576 return VINF_SUCCESS;
2577}
2578
2579
2580/**
2581 * Sets up miscellaneous (everything other than Pin & Processor-based
2582 * VM-execution) control fields in the VMCS.
2583 *
2584 * @returns VBox status code.
2585 * @param pVCpu The cross context virtual CPU structure.
2586 */
2587static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2588{
2589 AssertPtr(pVCpu);
2590
2591 int rc = VERR_GENERAL_FAILURE;
2592
2593 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2594#if 0
2595 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2596 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2597 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2598
2599 /*
2600 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2601 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2602 * We thus use the exception bitmap to control it rather than use both.
2603 */
2604 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2605 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2606
2607 /* All IO & IOIO instructions cause VM-exits. */
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2609 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2610
2611 /* Initialize the MSR-bitmap area. */
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2614 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2615 AssertRCReturn(rc, rc);
2616#endif
2617
2618 /* Setup MSR auto-load/store area. */
2619 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2620 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2623 AssertRCReturn(rc, rc);
2624
2625 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2626 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2627 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2628 AssertRCReturn(rc, rc);
2629
2630 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2632 AssertRCReturn(rc, rc);
2633
2634 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2635#if 0
2636 /* Setup debug controls */
2637 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2638 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2639 AssertRCReturn(rc, rc);
2640#endif
2641
2642 return rc;
2643}
2644
2645
2646/**
2647 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2648 *
2649 * We shall setup those exception intercepts that don't change during the
2650 * lifetime of the VM here. The rest are done dynamically while loading the
2651 * guest state.
2652 *
2653 * @returns VBox status code.
2654 * @param pVCpu The cross context virtual CPU structure.
2655 */
2656static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2657{
2658 AssertPtr(pVCpu);
2659
2660 uint32_t uXcptBitmap;
2661
2662 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2663 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2664
2665 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2666 and writes, and because recursive #DBs can cause the CPU hang, we must always
2667 intercept #DB. */
2668 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2669
2670 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2671 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2672 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2673
2674 /* Commit it to the VMCS. */
2675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2676 AssertRCReturn(rc, rc);
2677
2678 /* Update our cache of the exception bitmap. */
2679 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2680 return VINF_SUCCESS;
2681}
2682
2683
2684/**
2685 * Does per-VM VT-x initialization.
2686 *
2687 * @returns VBox status code.
2688 * @param pVM The cross context VM structure.
2689 */
2690VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2691{
2692 LogFlowFunc(("pVM=%p\n", pVM));
2693
2694 int rc = hmR0VmxStructsAlloc(pVM);
2695 if (RT_FAILURE(rc))
2696 {
2697 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2698 return rc;
2699 }
2700
2701 return VINF_SUCCESS;
2702}
2703
2704
2705/**
2706 * Does per-VM VT-x termination.
2707 *
2708 * @returns VBox status code.
2709 * @param pVM The cross context VM structure.
2710 */
2711VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2712{
2713 LogFlowFunc(("pVM=%p\n", pVM));
2714
2715#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2716 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2717 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2718#endif
2719 hmR0VmxStructsFree(pVM);
2720 return VINF_SUCCESS;
2721}
2722
2723
2724/**
2725 * Sets up the VM for execution under VT-x.
2726 * This function is only called once per-VM during initialization.
2727 *
2728 * @returns VBox status code.
2729 * @param pVM The cross context VM structure.
2730 */
2731VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2732{
2733 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2734 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2735
2736 LogFlowFunc(("pVM=%p\n", pVM));
2737
2738 /*
2739 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2740 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2741 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2742 */
2743 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2744 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2745 || !pVM->hm.s.vmx.pRealModeTSS))
2746 {
2747 LogRelFunc(("Invalid real-on-v86 state.\n"));
2748 return VERR_INTERNAL_ERROR;
2749 }
2750
2751 /* Initialize these always, see hmR3InitFinalizeR0().*/
2752 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2753 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2754
2755 /* Setup the tagged-TLB flush handlers. */
2756 int rc = hmR0VmxSetupTaggedTlb(pVM);
2757 if (RT_FAILURE(rc))
2758 {
2759 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2760 return rc;
2761 }
2762
2763 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2764 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2765#if HC_ARCH_BITS == 64
2766 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2768 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2769 {
2770 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2771 }
2772#endif
2773
2774 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2775 RTCCUINTREG uHostCR4 = ASMGetCR4();
2776 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2777 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2778
2779 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2780 {
2781 PVMCPU pVCpu = &pVM->aCpus[i];
2782 AssertPtr(pVCpu);
2783 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2784
2785 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2786 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2787
2788 /* Set revision dword at the beginning of the VMCS structure. */
2789 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2790
2791 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2792 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 /* Load this VMCS as the current VMCS. */
2797 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2799 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801 rc = hmR0VmxSetupPinCtls(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804
2805 rc = hmR0VmxSetupProcCtls(pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2807 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808
2809 rc = hmR0VmxSetupMiscCtls(pVCpu);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 rc = hmR0VmxInitXcptBitmap(pVCpu);
2814 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2816
2817#if HC_ARCH_BITS == 32
2818 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2819 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2820 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2821#endif
2822
2823 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2824 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2825 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2826 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2827
2828 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2829
2830 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2831 }
2832
2833 return VINF_SUCCESS;
2834}
2835
2836
2837/**
2838 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2839 * the VMCS.
2840 *
2841 * @returns VBox status code.
2842 */
2843static int hmR0VmxExportHostControlRegs(void)
2844{
2845 RTCCUINTREG uReg = ASMGetCR0();
2846 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2847 AssertRCReturn(rc, rc);
2848
2849 uReg = ASMGetCR3();
2850 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2851 AssertRCReturn(rc, rc);
2852
2853 uReg = ASMGetCR4();
2854 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2855 AssertRCReturn(rc, rc);
2856 return rc;
2857}
2858
2859
2860/**
2861 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2862 * the host-state area in the VMCS.
2863 *
2864 * @returns VBox status code.
2865 * @param pVCpu The cross context virtual CPU structure.
2866 */
2867static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2868{
2869#if HC_ARCH_BITS == 64
2870/**
2871 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2872 * requirements. See hmR0VmxExportHostSegmentRegs().
2873 */
2874# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2875 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2876 { \
2877 bool fValidSelector = true; \
2878 if ((selValue) & X86_SEL_LDT) \
2879 { \
2880 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2881 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2882 } \
2883 if (fValidSelector) \
2884 { \
2885 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2886 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2887 } \
2888 (selValue) = 0; \
2889 }
2890
2891 /*
2892 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2893 * should -not- save the messed up state without restoring the original host-state,
2894 * see @bugref{7240}.
2895 *
2896 * This apparently can happen (most likely the FPU changes), deal with it rather than
2897 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2898 */
2899 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2900 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2901 {
2902 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2903 pVCpu->idCpu));
2904 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2905 }
2906 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2907#else
2908 RT_NOREF(pVCpu);
2909#endif
2910
2911 /*
2912 * Host DS, ES, FS and GS segment registers.
2913 */
2914#if HC_ARCH_BITS == 64
2915 RTSEL uSelDS = ASMGetDS();
2916 RTSEL uSelES = ASMGetES();
2917 RTSEL uSelFS = ASMGetFS();
2918 RTSEL uSelGS = ASMGetGS();
2919#else
2920 RTSEL uSelDS = 0;
2921 RTSEL uSelES = 0;
2922 RTSEL uSelFS = 0;
2923 RTSEL uSelGS = 0;
2924#endif
2925
2926 /*
2927 * Host CS and SS segment registers.
2928 */
2929 RTSEL uSelCS = ASMGetCS();
2930 RTSEL uSelSS = ASMGetSS();
2931
2932 /*
2933 * Host TR segment register.
2934 */
2935 RTSEL uSelTR = ASMGetTR();
2936
2937#if HC_ARCH_BITS == 64
2938 /*
2939 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2940 * gain VM-entry and restore them before we get preempted.
2941 *
2942 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2943 */
2944 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2945 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2946 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2947 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2948# undef VMXLOCAL_ADJUST_HOST_SEG
2949#endif
2950
2951 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2952 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2953 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2954 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2955 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2956 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2957 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2958 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2959 Assert(uSelCS);
2960 Assert(uSelTR);
2961
2962 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2963#if 0
2964 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2965 Assert(uSelSS != 0);
2966#endif
2967
2968 /* Write these host selector fields into the host-state area in the VMCS. */
2969 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2970 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2971#if HC_ARCH_BITS == 64
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2975 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2976#else
2977 NOREF(uSelDS);
2978 NOREF(uSelES);
2979 NOREF(uSelFS);
2980 NOREF(uSelGS);
2981#endif
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2983 AssertRCReturn(rc, rc);
2984
2985 /*
2986 * Host GDTR and IDTR.
2987 */
2988 RTGDTR Gdtr;
2989 RTIDTR Idtr;
2990 RT_ZERO(Gdtr);
2991 RT_ZERO(Idtr);
2992 ASMGetGDTR(&Gdtr);
2993 ASMGetIDTR(&Idtr);
2994 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2995 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2996 AssertRCReturn(rc, rc);
2997
2998#if HC_ARCH_BITS == 64
2999 /*
3000 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3001 * them to the maximum limit (0xffff) on every VM-exit.
3002 */
3003 if (Gdtr.cbGdt != 0xffff)
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3005
3006 /*
3007 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3008 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3009 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3010 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3011 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3012 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3013 * at 0xffff on hosts where we are sure it won't cause trouble.
3014 */
3015# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3016 if (Idtr.cbIdt < 0x0fff)
3017# else
3018 if (Idtr.cbIdt != 0xffff)
3019# endif
3020 {
3021 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3022 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3023 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3024 }
3025#endif
3026
3027 /*
3028 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3029 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3030 * RPL should be too in most cases.
3031 */
3032 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3033 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3034
3035 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3036#if HC_ARCH_BITS == 64
3037 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3038
3039 /*
3040 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3041 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3042 * restoration if the host has something else. Task switching is not supported in 64-bit
3043 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3044 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3045 *
3046 * [1] See Intel spec. 3.5 "System Descriptor Types".
3047 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3048 */
3049 PVM pVM = pVCpu->CTX_SUFF(pVM);
3050 Assert(pDesc->System.u4Type == 11);
3051 if ( pDesc->System.u16LimitLow != 0x67
3052 || pDesc->System.u4LimitHigh)
3053 {
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3055 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3056 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3057 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3058 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3059 }
3060
3061 /*
3062 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3063 */
3064 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3065 {
3066 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3067 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3068 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3069 {
3070 /* The GDT is read-only but the writable GDT is available. */
3071 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3072 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3073 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3074 AssertRCReturn(rc, rc);
3075 }
3076 }
3077#else
3078 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3079#endif
3080 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3081 AssertRCReturn(rc, rc);
3082
3083 /*
3084 * Host FS base and GS base.
3085 */
3086#if HC_ARCH_BITS == 64
3087 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3088 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3089 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3090 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3094 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3095 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3096 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3097 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3098#endif
3099 return VINF_SUCCESS;
3100}
3101
3102
3103/**
3104 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3105 * host-state area of the VMCS.
3106 *
3107 * Theses MSRs will be automatically restored on the host after every successful
3108 * VM-exit.
3109 *
3110 * @returns VBox status code.
3111 * @param pVCpu The cross context virtual CPU structure.
3112 *
3113 * @remarks No-long-jump zone!!!
3114 */
3115static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3116{
3117 AssertPtr(pVCpu);
3118 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3119
3120 /*
3121 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3122 * rather than swapping them on every VM-entry.
3123 */
3124 hmR0VmxLazySaveHostMsrs(pVCpu);
3125
3126 /*
3127 * Host Sysenter MSRs.
3128 */
3129 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3130#if HC_ARCH_BITS == 32
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3132 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3133#else
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3135 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3136#endif
3137 AssertRCReturn(rc, rc);
3138
3139 /*
3140 * Host EFER MSR.
3141 *
3142 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3143 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3144 */
3145 PVM pVM = pVCpu->CTX_SUFF(pVM);
3146 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3147 {
3148 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3149 AssertRCReturn(rc, rc);
3150 }
3151
3152 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3153
3154 return VINF_SUCCESS;
3155}
3156
3157
3158/**
3159 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3160 *
3161 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3162 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3163 * hmR0VMxExportGuestEntryCtls().
3164 *
3165 * @returns true if we need to load guest EFER, false otherwise.
3166 * @param pVCpu The cross context virtual CPU structure.
3167 *
3168 * @remarks Requires EFER, CR4.
3169 * @remarks No-long-jump zone!!!
3170 */
3171static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3172{
3173#ifdef HMVMX_ALWAYS_SWAP_EFER
3174 return true;
3175#endif
3176
3177 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3178#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3179 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3180 if (CPUMIsGuestInLongModeEx(pCtx))
3181 return false;
3182#endif
3183
3184 PVM pVM = pVCpu->CTX_SUFF(pVM);
3185 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3186 uint64_t const u64GuestEfer = pCtx->msrEFER;
3187
3188 /*
3189 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3190 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3191 */
3192 if ( CPUMIsGuestInLongModeEx(pCtx)
3193 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3194 {
3195 return true;
3196 }
3197
3198 /*
3199 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3200 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3201 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3202 */
3203 if ( (pCtx->cr4 & X86_CR4_PAE)
3204 && (pCtx->cr0 & X86_CR0_PG)
3205 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3206 {
3207 /* Assert that host is PAE capable. */
3208 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3209 return true;
3210 }
3211
3212 return false;
3213}
3214
3215
3216/**
3217 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3218 *
3219 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3220 * see Intel spec. 24.8.1 "VM-entry controls".
3221 *
3222 * @returns VBox status code.
3223 * @param pVCpu The cross context virtual CPU structure.
3224 *
3225 * @remarks Requires EFER.
3226 * @remarks No-long-jump zone!!!
3227 */
3228static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3229{
3230 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3231 {
3232 PVM pVM = pVCpu->CTX_SUFF(pVM);
3233 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3234 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3235
3236 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3237 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3238
3239 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3240 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3241 {
3242 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3243 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3244 }
3245 else
3246 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3247
3248 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3249 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3250 && hmR0VmxShouldSwapEferMsr(pVCpu))
3251 {
3252 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3253 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3254 }
3255
3256 /*
3257 * The following should -not- be set (since we're not in SMM mode):
3258 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3259 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3260 */
3261
3262 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3263 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3264
3265 if ((fVal & fZap) != fVal)
3266 {
3267 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3268 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3269 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3270 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3271 }
3272
3273 /* Commit it to the VMCS and update our cache. */
3274 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3275 {
3276 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3277 AssertRCReturn(rc, rc);
3278 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3279 }
3280
3281 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3282 }
3283 return VINF_SUCCESS;
3284}
3285
3286
3287/**
3288 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3289 *
3290 * @returns VBox status code.
3291 * @param pVCpu The cross context virtual CPU structure.
3292 *
3293 * @remarks Requires EFER.
3294 */
3295static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3296{
3297 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3298 {
3299 PVM pVM = pVCpu->CTX_SUFF(pVM);
3300 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3301 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3302
3303 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3304 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3305
3306 /*
3307 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3308 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3309 * hmR0VmxExportHostMsrs().
3310 */
3311#if HC_ARCH_BITS == 64
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314#else
3315 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3316 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3317 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3318 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3319 {
3320 /* The switcher returns to long mode, EFER is managed by the switcher. */
3321 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3322 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3323 }
3324 else
3325 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3326#endif
3327
3328 /* If the newer VMCS fields for managing EFER exists, use it. */
3329 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3330 && hmR0VmxShouldSwapEferMsr(pVCpu))
3331 {
3332 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3333 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3334 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3335 }
3336
3337 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3338 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3339
3340 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3341 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3342 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3343
3344 /* Enable saving of the VMX preemption timer value on VM-exit. */
3345 if ( pVM->hm.s.vmx.fUsePreemptTimer
3346 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3347 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3348
3349 if ((fVal & fZap) != fVal)
3350 {
3351 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3352 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3353 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3354 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3355 }
3356
3357 /* Commit it to the VMCS and update our cache. */
3358 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3359 {
3360 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3361 AssertRCReturn(rc, rc);
3362 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3363 }
3364
3365 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3366 }
3367 return VINF_SUCCESS;
3368}
3369
3370
3371/**
3372 * Sets the TPR threshold in the VMCS.
3373 *
3374 * @returns VBox status code.
3375 * @param pVCpu The cross context virtual CPU structure.
3376 * @param u32TprThreshold The TPR threshold (task-priority class only).
3377 */
3378DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3379{
3380 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3381 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3382 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3383}
3384
3385
3386/**
3387 * Exports the guest APIC TPR state into the VMCS.
3388 *
3389 * @returns VBox status code.
3390 * @param pVCpu The cross context virtual CPU structure.
3391 *
3392 * @remarks No-long-jump zone!!!
3393 */
3394static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3395{
3396 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3397 {
3398 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3399
3400 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3401 && APICIsEnabled(pVCpu))
3402 {
3403 /*
3404 * Setup TPR shadowing.
3405 */
3406 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3407 {
3408 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3409
3410 bool fPendingIntr = false;
3411 uint8_t u8Tpr = 0;
3412 uint8_t u8PendingIntr = 0;
3413 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3414 AssertRCReturn(rc, rc);
3415
3416 /*
3417 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3418 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3419 * priority of the pending interrupt so we can deliver the interrupt. If there
3420 * are no interrupts pending, set threshold to 0 to not cause any
3421 * TPR-below-threshold VM-exits.
3422 */
3423 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3424 uint32_t u32TprThreshold = 0;
3425 if (fPendingIntr)
3426 {
3427 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3428 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3429 const uint8_t u8TprPriority = u8Tpr >> 4;
3430 if (u8PendingPriority <= u8TprPriority)
3431 u32TprThreshold = u8PendingPriority;
3432 }
3433
3434 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3435 AssertRCReturn(rc, rc);
3436 }
3437 }
3438 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3439 }
3440 return VINF_SUCCESS;
3441}
3442
3443
3444/**
3445 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3446 *
3447 * @returns Guest's interruptibility-state.
3448 * @param pVCpu The cross context virtual CPU structure.
3449 *
3450 * @remarks No-long-jump zone!!!
3451 */
3452static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3453{
3454 /*
3455 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3456 */
3457 uint32_t fIntrState = 0;
3458 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3459 {
3460 /* If inhibition is active, RIP & RFLAGS should've been accessed
3461 (i.e. read previously from the VMCS or from ring-3). */
3462 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3463#ifdef VBOX_STRICT
3464 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3465 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3466#endif
3467 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3468 {
3469 if (pCtx->eflags.Bits.u1IF)
3470 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3471 else
3472 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3473 }
3474 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3475 {
3476 /*
3477 * We can clear the inhibit force flag as even if we go back to the recompiler
3478 * without executing guest code in VT-x, the flag's condition to be cleared is
3479 * met and thus the cleared state is correct.
3480 */
3481 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3482 }
3483 }
3484
3485 /*
3486 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3487 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3488 * setting this would block host-NMIs and IRET will not clear the blocking.
3489 *
3490 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3491 */
3492 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3493 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3494 {
3495 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3496 }
3497
3498 return fIntrState;
3499}
3500
3501
3502/**
3503 * Exports the guest's interruptibility-state into the guest-state area in the
3504 * VMCS.
3505 *
3506 * @returns VBox status code.
3507 * @param pVCpu The cross context virtual CPU structure.
3508 * @param fIntrState The interruptibility-state to set.
3509 */
3510static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3511{
3512 NOREF(pVCpu);
3513 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3514 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3515 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3516}
3517
3518
3519/**
3520 * Exports the exception intercepts required for guest execution in the VMCS.
3521 *
3522 * @returns VBox status code.
3523 * @param pVCpu The cross context virtual CPU structure.
3524 *
3525 * @remarks No-long-jump zone!!!
3526 */
3527static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3528{
3529 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3530 {
3531 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3532
3533 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3534 if (pVCpu->hm.s.fGIMTrapXcptUD)
3535 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3536#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3537 else
3538 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3539#endif
3540
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3542 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3543
3544 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3545 {
3546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3547 AssertRCReturn(rc, rc);
3548 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3549 }
3550
3551 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3552 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3553 }
3554 return VINF_SUCCESS;
3555}
3556
3557
3558/**
3559 * Exports the guest's RIP into the guest-state area in the VMCS.
3560 *
3561 * @returns VBox status code.
3562 * @param pVCpu The cross context virtual CPU structure.
3563 *
3564 * @remarks No-long-jump zone!!!
3565 */
3566static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3567{
3568 int rc = VINF_SUCCESS;
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3570 {
3571 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3572
3573 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3574 AssertRCReturn(rc, rc);
3575
3576 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3577 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3578 }
3579 return rc;
3580}
3581
3582
3583/**
3584 * Exports the guest's RSP into the guest-state area in the VMCS.
3585 *
3586 * @returns VBox status code.
3587 * @param pVCpu The cross context virtual CPU structure.
3588 *
3589 * @remarks No-long-jump zone!!!
3590 */
3591static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3592{
3593 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3594 {
3595 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3596
3597 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3598 AssertRCReturn(rc, rc);
3599
3600 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3601 }
3602 return VINF_SUCCESS;
3603}
3604
3605
3606/**
3607 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 *
3612 * @remarks No-long-jump zone!!!
3613 */
3614static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3615{
3616 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617 {
3618 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3619
3620 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3621 Let us assert it as such and use 32-bit VMWRITE. */
3622 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3623 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3624 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3625 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3626
3627 /*
3628 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3629 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3630 * can run the real-mode guest code under Virtual 8086 mode.
3631 */
3632 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3633 {
3634 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3635 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3636 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3637 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3638 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3639 }
3640
3641 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3642 AssertRCReturn(rc, rc);
3643
3644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3645 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3646 }
3647 return VINF_SUCCESS;
3648}
3649
3650
3651/**
3652 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3653 *
3654 * The guest FPU state is always pre-loaded hence we don't need to bother about
3655 * sharing FPU related CR0 bits between the guest and host.
3656 *
3657 * @returns VBox status code.
3658 * @param pVCpu The cross context virtual CPU structure.
3659 *
3660 * @remarks No-long-jump zone!!!
3661 */
3662static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3663{
3664 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3665 {
3666 PVM pVM = pVCpu->CTX_SUFF(pVM);
3667 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3668 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3669
3670 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3671 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3672
3673 /*
3674 * Setup VT-x's view of the guest CR0.
3675 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3676 */
3677 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3678 if (pVM->hm.s.fNestedPaging)
3679 {
3680 if (CPUMIsGuestPagingEnabled(pVCpu))
3681 {
3682 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3683 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3684 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3685 }
3686 else
3687 {
3688 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3689 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3690 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692
3693 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3694 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3695 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3696 }
3697 else
3698 {
3699 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3700 u32GuestCr0 |= X86_CR0_WP;
3701 }
3702
3703 /*
3704 * Guest FPU bits.
3705 *
3706 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3707 * using CR0.TS.
3708 *
3709 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3710 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3711 */
3712 u32GuestCr0 |= X86_CR0_NE;
3713
3714 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3715 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3716
3717 /*
3718 * Update exception intercepts.
3719 */
3720 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3721 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3722 {
3723 Assert(PDMVmmDevHeapIsEnabled(pVM));
3724 Assert(pVM->hm.s.vmx.pRealModeTSS);
3725 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3726 }
3727 else
3728 {
3729 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3730 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3731 if (fInterceptMF)
3732 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3733 }
3734
3735 /* Additional intercepts for debugging, define these yourself explicitly. */
3736#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3737 uXcptBitmap |= 0
3738 | RT_BIT(X86_XCPT_BP)
3739 | RT_BIT(X86_XCPT_DE)
3740 | RT_BIT(X86_XCPT_NM)
3741 | RT_BIT(X86_XCPT_TS)
3742 | RT_BIT(X86_XCPT_UD)
3743 | RT_BIT(X86_XCPT_NP)
3744 | RT_BIT(X86_XCPT_SS)
3745 | RT_BIT(X86_XCPT_GP)
3746 | RT_BIT(X86_XCPT_PF)
3747 | RT_BIT(X86_XCPT_MF)
3748 ;
3749#elif defined(HMVMX_ALWAYS_TRAP_PF)
3750 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3751#endif
3752 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 u32GuestCr0 |= fSetCr0;
3765 u32GuestCr0 &= fZapCr0;
3766 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t u32Cr0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 u32Cr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3793 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3798 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3799 AssertRCReturn(rc, rc);
3800
3801 /* Update our caches. */
3802 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3803 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3804 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3805
3806 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3807
3808 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3809 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3810 }
3811
3812 return VINF_SUCCESS;
3813}
3814
3815
3816/**
3817 * Exports the guest control registers (CR3, CR4) into the guest-state area
3818 * in the VMCS.
3819 *
3820 * @returns VBox strict status code.
3821 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3822 * without unrestricted guest access and the VMMDev is not presently
3823 * mapped (e.g. EFI32).
3824 *
3825 * @param pVCpu The cross context virtual CPU structure.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3845
3846 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3847 if (pVM->hm.s.fNestedPaging)
3848 {
3849 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3850
3851 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3852 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3854 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3855
3856 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3857 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3858 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3859
3860 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3861 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3862 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3863 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3864 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3865 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3866 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3867
3868 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3869 AssertRCReturn(rc, rc);
3870
3871 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3872 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3873 || CPUMIsGuestPagingEnabledEx(pCtx))
3874 {
3875 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3876 if (CPUMIsGuestInPAEModeEx(pCtx))
3877 {
3878 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3879 AssertRCReturn(rc, rc);
3880 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3883 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3884 AssertRCReturn(rc, rc);
3885 }
3886
3887 /*
3888 * The guest's view of its CR3 is unblemished with Nested Paging when the
3889 * guest is using paging or we have unrestricted guest execution to handle
3890 * the guest when it's not using paging.
3891 */
3892 GCPhysGuestCR3 = pCtx->cr3;
3893 }
3894 else
3895 {
3896 /*
3897 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3898 * thinks it accesses physical memory directly, we use our identity-mapped
3899 * page table to map guest-linear to guest-physical addresses. EPT takes care
3900 * of translating it to host-physical addresses.
3901 */
3902 RTGCPHYS GCPhys;
3903 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3904
3905 /* We obtain it here every time as the guest could have relocated this PCI region. */
3906 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3907 if (RT_SUCCESS(rc))
3908 { /* likely */ }
3909 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3910 {
3911 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3912 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3913 }
3914 else
3915 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3916
3917 GCPhysGuestCR3 = GCPhys;
3918 }
3919
3920 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3921 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3922 AssertRCReturn(rc, rc);
3923 }
3924 else
3925 {
3926 /* Non-nested paging case, just use the hypervisor's CR3. */
3927 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3928
3929 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3930 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3931 AssertRCReturn(rc, rc);
3932 }
3933
3934 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3935 }
3936
3937 /*
3938 * Guest CR4.
3939 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3940 */
3941 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3942 {
3943 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3944 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3945 Assert(!RT_HI_U32(pCtx->cr4));
3946
3947 uint32_t u32GuestCr4 = pCtx->cr4;
3948 uint32_t const u32ShadowCr4 = pCtx->cr4;
3949
3950 /*
3951 * Setup VT-x's view of the guest CR4.
3952 *
3953 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3954 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3955 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3956 *
3957 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3958 */
3959 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3960 {
3961 Assert(pVM->hm.s.vmx.pRealModeTSS);
3962 Assert(PDMVmmDevHeapIsEnabled(pVM));
3963 u32GuestCr4 &= ~X86_CR4_VME;
3964 }
3965
3966 if (pVM->hm.s.fNestedPaging)
3967 {
3968 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3969 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3970 {
3971 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3972 u32GuestCr4 |= X86_CR4_PSE;
3973 /* Our identity mapping is a 32-bit page directory. */
3974 u32GuestCr4 &= ~X86_CR4_PAE;
3975 }
3976 /* else use guest CR4.*/
3977 }
3978 else
3979 {
3980 /*
3981 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3982 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3983 */
3984 switch (pVCpu->hm.s.enmShadowMode)
3985 {
3986 case PGMMODE_REAL: /* Real-mode. */
3987 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3988 case PGMMODE_32_BIT: /* 32-bit paging. */
3989 {
3990 u32GuestCr4 &= ~X86_CR4_PAE;
3991 break;
3992 }
3993
3994 case PGMMODE_PAE: /* PAE paging. */
3995 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3996 {
3997 u32GuestCr4 |= X86_CR4_PAE;
3998 break;
3999 }
4000
4001 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4002 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4003#ifdef VBOX_ENABLE_64_BITS_GUESTS
4004 break;
4005#endif
4006 default:
4007 AssertFailed();
4008 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4009 }
4010 }
4011
4012 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4013 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4015 u32GuestCr4 |= fSetCr4;
4016 u32GuestCr4 &= fZapCr4;
4017
4018 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4019 that would cause a VM-exit. */
4020 uint32_t u32Cr4Mask = X86_CR4_VME
4021 | X86_CR4_PAE
4022 | X86_CR4_PGE
4023 | X86_CR4_PSE
4024 | X86_CR4_VMXE;
4025 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4026 u32Cr4Mask |= X86_CR4_OSXSAVE;
4027 if (pVM->cpum.ro.GuestFeatures.fPcid)
4028 u32Cr4Mask |= X86_CR4_PCIDE;
4029
4030 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4031 into the VMCS and update our cache. */
4032 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4034 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4035 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4036 AssertRCReturn(rc, rc);
4037 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4038
4039 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4040 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4041
4042 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4043
4044 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4045 fZapCr4));
4046 }
4047 return rc;
4048}
4049
4050
4051/**
4052 * Exports the guest debug registers into the guest-state area in the VMCS.
4053 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4054 *
4055 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4056 *
4057 * @returns VBox status code.
4058 * @param pVCpu The cross context virtual CPU structure.
4059 *
4060 * @remarks No-long-jump zone!!!
4061 */
4062static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4063{
4064 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4065
4066#ifdef VBOX_STRICT
4067 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4068 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4069 {
4070 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4072 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4073 }
4074#endif
4075
4076 bool fSteppingDB = false;
4077 bool fInterceptMovDRx = false;
4078 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4079 if (pVCpu->hm.s.fSingleInstruction)
4080 {
4081 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4082 PVM pVM = pVCpu->CTX_SUFF(pVM);
4083 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4084 {
4085 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4086 Assert(fSteppingDB == false);
4087 }
4088 else
4089 {
4090 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4091 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4092 pVCpu->hm.s.fClearTrapFlag = true;
4093 fSteppingDB = true;
4094 }
4095 }
4096
4097 uint32_t u32GuestDr7;
4098 if ( fSteppingDB
4099 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4100 {
4101 /*
4102 * Use the combined guest and host DRx values found in the hypervisor register set
4103 * because the debugger has breakpoints active or someone is single stepping on the
4104 * host side without a monitor trap flag.
4105 *
4106 * Note! DBGF expects a clean DR6 state before executing guest code.
4107 */
4108#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4109 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4110 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4111 {
4112 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4113 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4114 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4115 }
4116 else
4117#endif
4118 if (!CPUMIsHyperDebugStateActive(pVCpu))
4119 {
4120 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4121 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4122 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4123 }
4124
4125 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4126 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4127 pVCpu->hm.s.fUsingHyperDR7 = true;
4128 fInterceptMovDRx = true;
4129 }
4130 else
4131 {
4132 /*
4133 * If the guest has enabled debug registers, we need to load them prior to
4134 * executing guest code so they'll trigger at the right time.
4135 */
4136 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4137 {
4138#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4139 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4140 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4141 {
4142 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4143 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4144 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4145 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4146 }
4147 else
4148#endif
4149 if (!CPUMIsGuestDebugStateActive(pVCpu))
4150 {
4151 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4152 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4153 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4154 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4155 }
4156 Assert(!fInterceptMovDRx);
4157 }
4158 /*
4159 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4160 * must intercept #DB in order to maintain a correct DR6 guest value, and
4161 * because we need to intercept it to prevent nested #DBs from hanging the
4162 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4163 */
4164#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4165 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4166 && !CPUMIsGuestDebugStateActive(pVCpu))
4167#else
4168 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4169#endif
4170 {
4171 fInterceptMovDRx = true;
4172 }
4173
4174 /* Update DR7 with the actual guest value. */
4175 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4176 pVCpu->hm.s.fUsingHyperDR7 = false;
4177 }
4178
4179 if (fInterceptMovDRx)
4180 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4181 else
4182 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4183
4184 /*
4185 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4186 * monitor-trap flag and update our cache.
4187 */
4188 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4189 {
4190 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4191 AssertRCReturn(rc2, rc2);
4192 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4193 }
4194
4195 /*
4196 * Update guest DR7.
4197 */
4198 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4199 AssertRCReturn(rc, rc);
4200
4201 return VINF_SUCCESS;
4202}
4203
4204
4205#ifdef VBOX_STRICT
4206/**
4207 * Strict function to validate segment registers.
4208 *
4209 * @param pVCpu The cross context virtual CPU structure.
4210 * @param pCtx Pointer to the guest-CPU context.
4211 *
4212 * @remarks Will import guest CR0 on strict builds during validation of
4213 * segments.
4214 */
4215static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4216{
4217 /*
4218 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4219 *
4220 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4221 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4222 * and doesn't change the guest-context value.
4223 */
4224 PVM pVM = pVCpu->CTX_SUFF(pVM);
4225 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4226 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4227 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4228 && ( !CPUMIsGuestInRealModeEx(pCtx)
4229 && !CPUMIsGuestInV86ModeEx(pCtx)))
4230 {
4231 /* Protected mode checks */
4232 /* CS */
4233 Assert(pCtx->cs.Attr.n.u1Present);
4234 Assert(!(pCtx->cs.Attr.u & 0xf00));
4235 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->cs.Attr.n.u1Granularity));
4238 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4239 || (pCtx->cs.Attr.n.u1Granularity));
4240 /* CS cannot be loaded with NULL in protected mode. */
4241 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4242 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4243 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4244 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4245 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4246 else
4247 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4248 /* SS */
4249 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4250 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4251 if ( !(pCtx->cr0 & X86_CR0_PE)
4252 || pCtx->cs.Attr.n.u4Type == 3)
4253 {
4254 Assert(!pCtx->ss.Attr.n.u2Dpl);
4255 }
4256 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4259 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4260 Assert(pCtx->ss.Attr.n.u1Present);
4261 Assert(!(pCtx->ss.Attr.u & 0xf00));
4262 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->ss.Attr.n.u1Granularity));
4265 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4266 || (pCtx->ss.Attr.n.u1Granularity));
4267 }
4268 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4269 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4270 {
4271 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4272 Assert(pCtx->ds.Attr.n.u1Present);
4273 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4274 Assert(!(pCtx->ds.Attr.u & 0xf00));
4275 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4276 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4277 || !(pCtx->ds.Attr.n.u1Granularity));
4278 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4279 || (pCtx->ds.Attr.n.u1Granularity));
4280 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4281 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4282 }
4283 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4284 {
4285 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4286 Assert(pCtx->es.Attr.n.u1Present);
4287 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4288 Assert(!(pCtx->es.Attr.u & 0xf00));
4289 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4290 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4291 || !(pCtx->es.Attr.n.u1Granularity));
4292 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4293 || (pCtx->es.Attr.n.u1Granularity));
4294 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4295 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4296 }
4297 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4298 {
4299 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4300 Assert(pCtx->fs.Attr.n.u1Present);
4301 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4302 Assert(!(pCtx->fs.Attr.u & 0xf00));
4303 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4304 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4305 || !(pCtx->fs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4307 || (pCtx->fs.Attr.n.u1Granularity));
4308 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4309 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4310 }
4311 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4312 {
4313 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4314 Assert(pCtx->gs.Attr.n.u1Present);
4315 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4316 Assert(!(pCtx->gs.Attr.u & 0xf00));
4317 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4318 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4319 || !(pCtx->gs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4321 || (pCtx->gs.Attr.n.u1Granularity));
4322 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4323 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4324 }
4325 /* 64-bit capable CPUs. */
4326# if HC_ARCH_BITS == 64
4327 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4328 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4329 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4330 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4331# endif
4332 }
4333 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4334 || ( CPUMIsGuestInRealModeEx(pCtx)
4335 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4336 {
4337 /* Real and v86 mode checks. */
4338 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4339 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4340 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4341 {
4342 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4343 }
4344 else
4345 {
4346 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4347 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4348 }
4349
4350 /* CS */
4351 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4352 Assert(pCtx->cs.u32Limit == 0xffff);
4353 Assert(u32CSAttr == 0xf3);
4354 /* SS */
4355 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4356 Assert(pCtx->ss.u32Limit == 0xffff);
4357 Assert(u32SSAttr == 0xf3);
4358 /* DS */
4359 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4360 Assert(pCtx->ds.u32Limit == 0xffff);
4361 Assert(u32DSAttr == 0xf3);
4362 /* ES */
4363 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4364 Assert(pCtx->es.u32Limit == 0xffff);
4365 Assert(u32ESAttr == 0xf3);
4366 /* FS */
4367 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4368 Assert(pCtx->fs.u32Limit == 0xffff);
4369 Assert(u32FSAttr == 0xf3);
4370 /* GS */
4371 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4372 Assert(pCtx->gs.u32Limit == 0xffff);
4373 Assert(u32GSAttr == 0xf3);
4374 /* 64-bit capable CPUs. */
4375# if HC_ARCH_BITS == 64
4376 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4377 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4378 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4379 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4380# endif
4381 }
4382}
4383#endif /* VBOX_STRICT */
4384
4385
4386/**
4387 * Exports a guest segment register into the guest-state area in the VMCS.
4388 *
4389 * @returns VBox status code.
4390 * @param pVCpu The cross context virtual CPU structure.
4391 * @param idxSel Index of the selector in the VMCS.
4392 * @param idxLimit Index of the segment limit in the VMCS.
4393 * @param idxBase Index of the segment base in the VMCS.
4394 * @param idxAccess Index of the access rights of the segment in the VMCS.
4395 * @param pSelReg Pointer to the segment selector.
4396 *
4397 * @remarks No-long-jump zone!!!
4398 */
4399static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4400 PCCPUMSELREG pSelReg)
4401{
4402 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4403 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4404 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4405 AssertRCReturn(rc, rc);
4406
4407 uint32_t u32Access = pSelReg->Attr.u;
4408 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4409 {
4410 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4411 u32Access = 0xf3;
4412 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4413 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4414 }
4415 else
4416 {
4417 /*
4418 * The way to differentiate between whether this is really a null selector or was just
4419 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4420 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4421 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4422 * NULL selectors loaded in protected-mode have their attribute as 0.
4423 */
4424 if (!u32Access)
4425 u32Access = X86DESCATTR_UNUSABLE;
4426 }
4427
4428 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4429 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4430 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4431
4432 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4433 AssertRCReturn(rc, rc);
4434 return rc;
4435}
4436
4437
4438/**
4439 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4440 * into the guest-state area in the VMCS.
4441 *
4442 * @returns VBox status code.
4443 * @param pVCpu The cross context virtual CPU structure.
4444 *
4445 * @remarks Will import guest CR0 on strict builds during validation of
4446 * segments.
4447 * @remarks No-long-jump zone!!!
4448 */
4449static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4450{
4451 int rc = VERR_INTERNAL_ERROR_5;
4452 PVM pVM = pVCpu->CTX_SUFF(pVM);
4453 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4454
4455 /*
4456 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4457 */
4458 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4459 {
4460#ifdef VBOX_WITH_REM
4461 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4462 {
4463 Assert(pVM->hm.s.vmx.pRealModeTSS);
4464 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4465 if ( pVCpu->hm.s.vmx.fWasInRealMode
4466 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4467 {
4468 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4469 in real-mode (e.g. OpenBSD 4.0) */
4470 REMFlushTBs(pVM);
4471 Log4Func(("Switch to protected mode detected!\n"));
4472 pVCpu->hm.s.vmx.fWasInRealMode = false;
4473 }
4474 }
4475#endif
4476 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4477 {
4478 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4479 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4480 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4481 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4482 AssertRCReturn(rc, rc);
4483 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4484 }
4485
4486 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4487 {
4488 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4489 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4490 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4491 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4492 AssertRCReturn(rc, rc);
4493 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4494 }
4495
4496 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4497 {
4498 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4499 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4500 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4501 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4502 AssertRCReturn(rc, rc);
4503 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4504 }
4505
4506 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4507 {
4508 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4509 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4510 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4511 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4512 AssertRCReturn(rc, rc);
4513 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4514 }
4515
4516 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4517 {
4518 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4519 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4520 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4521 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4522 AssertRCReturn(rc, rc);
4523 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4524 }
4525
4526 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4527 {
4528 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4529 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4530 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4531 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4532 AssertRCReturn(rc, rc);
4533 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4534 }
4535
4536#ifdef VBOX_STRICT
4537 hmR0VmxValidateSegmentRegs(pVCpu);
4538#endif
4539
4540 /* Update the exit history entry with the correct CS.BASE + RIP. */
4541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4542 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
4543
4544 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4545 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4546 }
4547
4548 /*
4549 * Guest TR.
4550 */
4551 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4552 {
4553 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4554
4555 /*
4556 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4557 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4558 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4559 */
4560 uint16_t u16Sel = 0;
4561 uint32_t u32Limit = 0;
4562 uint64_t u64Base = 0;
4563 uint32_t u32AccessRights = 0;
4564
4565 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4566 {
4567 u16Sel = pCtx->tr.Sel;
4568 u32Limit = pCtx->tr.u32Limit;
4569 u64Base = pCtx->tr.u64Base;
4570 u32AccessRights = pCtx->tr.Attr.u;
4571 }
4572 else
4573 {
4574 Assert(pVM->hm.s.vmx.pRealModeTSS);
4575 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4576
4577 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4578 RTGCPHYS GCPhys;
4579 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4580 AssertRCReturn(rc, rc);
4581
4582 X86DESCATTR DescAttr;
4583 DescAttr.u = 0;
4584 DescAttr.n.u1Present = 1;
4585 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4586
4587 u16Sel = 0;
4588 u32Limit = HM_VTX_TSS_SIZE;
4589 u64Base = GCPhys; /* in real-mode phys = virt. */
4590 u32AccessRights = DescAttr.u;
4591 }
4592
4593 /* Validate. */
4594 Assert(!(u16Sel & RT_BIT(2)));
4595 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4596 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4597 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4598 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4599 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4600 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4601 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4602 Assert( (u32Limit & 0xfff) == 0xfff
4603 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4604 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4605 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4606
4607 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4609 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4610 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4611 AssertRCReturn(rc, rc);
4612
4613 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4614 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4615 }
4616
4617 /*
4618 * Guest GDTR.
4619 */
4620 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4621 {
4622 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4623
4624 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4625 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4626 AssertRCReturn(rc, rc);
4627
4628 /* Validate. */
4629 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4630
4631 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4632 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4633 }
4634
4635 /*
4636 * Guest LDTR.
4637 */
4638 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4639 {
4640 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4641
4642 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4643 uint32_t u32Access = 0;
4644 if (!pCtx->ldtr.Attr.u)
4645 u32Access = X86DESCATTR_UNUSABLE;
4646 else
4647 u32Access = pCtx->ldtr.Attr.u;
4648
4649 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4650 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4651 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4652 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4653 AssertRCReturn(rc, rc);
4654
4655 /* Validate. */
4656 if (!(u32Access & X86DESCATTR_UNUSABLE))
4657 {
4658 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4659 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4660 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4661 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4662 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4663 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4664 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4665 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4666 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4667 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4668 }
4669
4670 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4671 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4672 }
4673
4674 /*
4675 * Guest IDTR.
4676 */
4677 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4678 {
4679 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4680
4681 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4682 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4683 AssertRCReturn(rc, rc);
4684
4685 /* Validate. */
4686 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4687
4688 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4689 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4690 }
4691
4692 return VINF_SUCCESS;
4693}
4694
4695
4696/**
4697 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4698 * areas.
4699 *
4700 * These MSRs will automatically be loaded to the host CPU on every successful
4701 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4702 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4703 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4704 *
4705 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4706 *
4707 * @returns VBox status code.
4708 * @param pVCpu The cross context virtual CPU structure.
4709 *
4710 * @remarks No-long-jump zone!!!
4711 */
4712static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4713{
4714 AssertPtr(pVCpu);
4715 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4716
4717 /*
4718 * MSRs that we use the auto-load/store MSR area in the VMCS.
4719 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4720 */
4721 PVM pVM = pVCpu->CTX_SUFF(pVM);
4722 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4723 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4724 {
4725 if (pVM->hm.s.fAllow64BitGuests)
4726 {
4727#if HC_ARCH_BITS == 32
4728 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4729
4730 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4731 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4732 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4733 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4734 AssertRCReturn(rc, rc);
4735# ifdef LOG_ENABLED
4736 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4737 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4738 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4739# endif
4740#endif
4741 }
4742 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4743 }
4744
4745 /*
4746 * Guest Sysenter MSRs.
4747 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4748 * VM-exits on WRMSRs for these MSRs.
4749 */
4750 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4751 {
4752 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4753
4754 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4755 {
4756 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4757 AssertRCReturn(rc, rc);
4758 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4759 }
4760
4761 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4762 {
4763 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4764 AssertRCReturn(rc, rc);
4765 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4766 }
4767
4768 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4769 {
4770 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4771 AssertRCReturn(rc, rc);
4772 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4773 }
4774 }
4775
4776 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4777 {
4778 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4779
4780 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4781 {
4782 /*
4783 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4784 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4785 */
4786 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4787 {
4788 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4789 AssertRCReturn(rc,rc);
4790 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4791 }
4792 else
4793 {
4794 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4795 NULL /* pfAddedAndUpdated */);
4796 AssertRCReturn(rc, rc);
4797
4798 /* We need to intercept reads too, see @bugref{7386#c16}. */
4799 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4800 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4801 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4802 pVCpu->hm.s.vmx.cMsrs));
4803 }
4804 }
4805 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4806 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4807 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4808 }
4809
4810 return VINF_SUCCESS;
4811}
4812
4813
4814#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4815/**
4816 * Check if guest state allows safe use of 32-bit switcher again.
4817 *
4818 * Segment bases and protected mode structures must be 32-bit addressable
4819 * because the 32-bit switcher will ignore high dword when writing these VMCS
4820 * fields. See @bugref{8432} for details.
4821 *
4822 * @returns true if safe, false if must continue to use the 64-bit switcher.
4823 * @param pCtx Pointer to the guest-CPU context.
4824 *
4825 * @remarks No-long-jump zone!!!
4826 */
4827static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4828{
4829 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4834 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4835 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4836 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4837 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4838 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4839
4840 /* All good, bases are 32-bit. */
4841 return true;
4842}
4843#endif
4844
4845
4846/**
4847 * Selects up the appropriate function to run guest code.
4848 *
4849 * @returns VBox status code.
4850 * @param pVCpu The cross context virtual CPU structure.
4851 *
4852 * @remarks No-long-jump zone!!!
4853 */
4854static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4855{
4856 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4857 if (CPUMIsGuestInLongModeEx(pCtx))
4858 {
4859#ifndef VBOX_ENABLE_64_BITS_GUESTS
4860 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4861#endif
4862 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4863#if HC_ARCH_BITS == 32
4864 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4865 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4866 {
4867#ifdef VBOX_STRICT
4868 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4869 {
4870 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4871 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4872 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4873 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4874 | HM_CHANGED_VMX_ENTRY_CTLS
4875 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4876 }
4877#endif
4878 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4879
4880 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4881 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4882 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4883 Log4Func(("Selected 64-bit switcher\n"));
4884 }
4885#else
4886 /* 64-bit host. */
4887 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4888#endif
4889 }
4890 else
4891 {
4892 /* Guest is not in long mode, use the 32-bit handler. */
4893#if HC_ARCH_BITS == 32
4894 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4895 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4896 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4897 {
4898# ifdef VBOX_STRICT
4899 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4900 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4901 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4902 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4903 | HM_CHANGED_VMX_ENTRY_CTLS
4904 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4905# endif
4906 }
4907# ifdef VBOX_ENABLE_64_BITS_GUESTS
4908 /*
4909 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4910 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4911 * switcher flag because now we know the guest is in a sane state where it's safe
4912 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4913 * the much faster 32-bit switcher again.
4914 */
4915 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4916 {
4917 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4918 Log4Func(("Selected 32-bit switcher\n"));
4919 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4920 }
4921 else
4922 {
4923 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4924 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4925 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4926 {
4927 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4928 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4929 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4930 | HM_CHANGED_VMX_ENTRY_CTLS
4931 | HM_CHANGED_VMX_EXIT_CTLS
4932 | HM_CHANGED_HOST_CONTEXT);
4933 Log4Func(("Selected 32-bit switcher (safe)\n"));
4934 }
4935 }
4936# else
4937 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4938# endif
4939#else
4940 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4941#endif
4942 }
4943 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4944 return VINF_SUCCESS;
4945}
4946
4947
4948/**
4949 * Wrapper for running the guest code in VT-x.
4950 *
4951 * @returns VBox status code, no informational status codes.
4952 * @param pVCpu The cross context virtual CPU structure.
4953 *
4954 * @remarks No-long-jump zone!!!
4955 */
4956DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4957{
4958 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4959 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4960 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4961
4962 /*
4963 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4964 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4965 * callee-saved and thus the need for this XMM wrapper.
4966 *
4967 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4968 */
4969 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4970 /** @todo Add stats for resume vs launch. */
4971 PVM pVM = pVCpu->CTX_SUFF(pVM);
4972#ifdef VBOX_WITH_KERNEL_USING_XMM
4973 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4974#else
4975 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4976#endif
4977 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4978 return rc;
4979}
4980
4981
4982/**
4983 * Reports world-switch error and dumps some useful debug info.
4984 *
4985 * @param pVCpu The cross context virtual CPU structure.
4986 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4987 * @param pVmxTransient Pointer to the VMX transient structure (only
4988 * exitReason updated).
4989 */
4990static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4991{
4992 Assert(pVCpu);
4993 Assert(pVmxTransient);
4994 HMVMX_ASSERT_PREEMPT_SAFE();
4995
4996 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4997 switch (rcVMRun)
4998 {
4999 case VERR_VMX_INVALID_VMXON_PTR:
5000 AssertFailed();
5001 break;
5002 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5003 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5004 {
5005 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5006 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5007 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5008 AssertRC(rc);
5009
5010 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5011 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5012 Cannot do it here as we may have been long preempted. */
5013
5014#ifdef VBOX_STRICT
5015 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5016 pVmxTransient->uExitReason));
5017 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5018 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5019 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5020 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5021 else
5022 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5023 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5024 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5025
5026 /* VMX control bits. */
5027 uint32_t u32Val;
5028 uint64_t u64Val;
5029 RTHCUINTREG uHCReg;
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5032 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5033 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5034 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5035 {
5036 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5037 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5038 }
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5045 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5046 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5047 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5048 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5049 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5050 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5065 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5066 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5067 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5068 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5069 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5070 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5071 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5072 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5073 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5074 {
5075 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5076 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5077 }
5078
5079 /* Guest bits. */
5080 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5081 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5082 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5083 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5084 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5085 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5086 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5087 {
5088 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5090 }
5091
5092 /* Host bits. */
5093 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5094 Log4(("Host CR0 %#RHr\n", uHCReg));
5095 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5096 Log4(("Host CR3 %#RHr\n", uHCReg));
5097 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5098 Log4(("Host CR4 %#RHr\n", uHCReg));
5099
5100 RTGDTR HostGdtr;
5101 PCX86DESCHC pDesc;
5102 ASMGetGDTR(&HostGdtr);
5103 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5104 Log4(("Host CS %#08x\n", u32Val));
5105 if (u32Val < HostGdtr.cbGdt)
5106 {
5107 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5108 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5109 }
5110
5111 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5112 Log4(("Host DS %#08x\n", u32Val));
5113 if (u32Val < HostGdtr.cbGdt)
5114 {
5115 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5116 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5117 }
5118
5119 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5120 Log4(("Host ES %#08x\n", u32Val));
5121 if (u32Val < HostGdtr.cbGdt)
5122 {
5123 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5124 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5125 }
5126
5127 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5128 Log4(("Host FS %#08x\n", u32Val));
5129 if (u32Val < HostGdtr.cbGdt)
5130 {
5131 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5132 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5133 }
5134
5135 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5136 Log4(("Host GS %#08x\n", u32Val));
5137 if (u32Val < HostGdtr.cbGdt)
5138 {
5139 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5140 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5141 }
5142
5143 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5144 Log4(("Host SS %#08x\n", u32Val));
5145 if (u32Val < HostGdtr.cbGdt)
5146 {
5147 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5148 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5149 }
5150
5151 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5152 Log4(("Host TR %#08x\n", u32Val));
5153 if (u32Val < HostGdtr.cbGdt)
5154 {
5155 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5156 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5157 }
5158
5159 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5160 Log4(("Host TR Base %#RHv\n", uHCReg));
5161 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5162 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5164 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5165 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5166 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5168 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5169 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5170 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5171 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5172 Log4(("Host RSP %#RHv\n", uHCReg));
5173 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5174 Log4(("Host RIP %#RHv\n", uHCReg));
5175# if HC_ARCH_BITS == 64
5176 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5177 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5178 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5179 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5180 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5181 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5182# endif
5183#endif /* VBOX_STRICT */
5184 break;
5185 }
5186
5187 default:
5188 /* Impossible */
5189 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5190 break;
5191 }
5192}
5193
5194
5195#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5196#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5197# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5198#endif
5199#ifdef VBOX_STRICT
5200static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5201{
5202 switch (idxField)
5203 {
5204 case VMX_VMCS_GUEST_RIP:
5205 case VMX_VMCS_GUEST_RSP:
5206 case VMX_VMCS_GUEST_SYSENTER_EIP:
5207 case VMX_VMCS_GUEST_SYSENTER_ESP:
5208 case VMX_VMCS_GUEST_GDTR_BASE:
5209 case VMX_VMCS_GUEST_IDTR_BASE:
5210 case VMX_VMCS_GUEST_CS_BASE:
5211 case VMX_VMCS_GUEST_DS_BASE:
5212 case VMX_VMCS_GUEST_ES_BASE:
5213 case VMX_VMCS_GUEST_FS_BASE:
5214 case VMX_VMCS_GUEST_GS_BASE:
5215 case VMX_VMCS_GUEST_SS_BASE:
5216 case VMX_VMCS_GUEST_LDTR_BASE:
5217 case VMX_VMCS_GUEST_TR_BASE:
5218 case VMX_VMCS_GUEST_CR3:
5219 return true;
5220 }
5221 return false;
5222}
5223
5224static bool hmR0VmxIsValidReadField(uint32_t idxField)
5225{
5226 switch (idxField)
5227 {
5228 /* Read-only fields. */
5229 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5230 return true;
5231 }
5232 /* Remaining readable fields should also be writable. */
5233 return hmR0VmxIsValidWriteField(idxField);
5234}
5235#endif /* VBOX_STRICT */
5236
5237
5238/**
5239 * Executes the specified handler in 64-bit mode.
5240 *
5241 * @returns VBox status code (no informational status codes).
5242 * @param pVCpu The cross context virtual CPU structure.
5243 * @param enmOp The operation to perform.
5244 * @param cParams Number of parameters.
5245 * @param paParam Array of 32-bit parameters.
5246 */
5247VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5248{
5249 PVM pVM = pVCpu->CTX_SUFF(pVM);
5250 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5251 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5252 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5253 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5254
5255#ifdef VBOX_STRICT
5256 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5257 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5258
5259 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5260 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5261#endif
5262
5263 /* Disable interrupts. */
5264 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5265
5266#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5267 RTCPUID idHostCpu = RTMpCpuId();
5268 CPUMR0SetLApic(pVCpu, idHostCpu);
5269#endif
5270
5271 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5272 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5273
5274 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5275 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5276 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5277
5278 /* Leave VMX Root Mode. */
5279 VMXDisable();
5280
5281 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5282
5283 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5284 CPUMSetHyperEIP(pVCpu, enmOp);
5285 for (int i = (int)cParams - 1; i >= 0; i--)
5286 CPUMPushHyper(pVCpu, paParam[i]);
5287
5288 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5289
5290 /* Call the switcher. */
5291 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5292 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5293
5294 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5295 /* Make sure the VMX instructions don't cause #UD faults. */
5296 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5297
5298 /* Re-enter VMX Root Mode */
5299 int rc2 = VMXEnable(HCPhysCpuPage);
5300 if (RT_FAILURE(rc2))
5301 {
5302 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5303 ASMSetFlags(fOldEFlags);
5304 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5305 return rc2;
5306 }
5307
5308 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5309 AssertRC(rc2);
5310 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5311 Assert(!(ASMGetFlags() & X86_EFL_IF));
5312 ASMSetFlags(fOldEFlags);
5313 return rc;
5314}
5315
5316
5317/**
5318 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5319 * supporting 64-bit guests.
5320 *
5321 * @returns VBox status code.
5322 * @param fResume Whether to VMLAUNCH or VMRESUME.
5323 * @param pCtx Pointer to the guest-CPU context.
5324 * @param pCache Pointer to the VMCS cache.
5325 * @param pVM The cross context VM structure.
5326 * @param pVCpu The cross context virtual CPU structure.
5327 */
5328DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5329{
5330 NOREF(fResume);
5331
5332 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5333 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5334
5335#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5336 pCache->uPos = 1;
5337 pCache->interPD = PGMGetInterPaeCR3(pVM);
5338 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5339#endif
5340
5341#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5342 pCache->TestIn.HCPhysCpuPage = 0;
5343 pCache->TestIn.HCPhysVmcs = 0;
5344 pCache->TestIn.pCache = 0;
5345 pCache->TestOut.HCPhysVmcs = 0;
5346 pCache->TestOut.pCache = 0;
5347 pCache->TestOut.pCtx = 0;
5348 pCache->TestOut.eflags = 0;
5349#else
5350 NOREF(pCache);
5351#endif
5352
5353 uint32_t aParam[10];
5354 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5355 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5356 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5357 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5358 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5359 aParam[5] = 0;
5360 aParam[6] = VM_RC_ADDR(pVM, pVM);
5361 aParam[7] = 0;
5362 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5363 aParam[9] = 0;
5364
5365#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5366 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5367 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5368#endif
5369 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5370
5371#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5372 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5373 Assert(pCtx->dr[4] == 10);
5374 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5375#endif
5376
5377#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5378 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5379 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5380 pVCpu->hm.s.vmx.HCPhysVmcs));
5381 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5382 pCache->TestOut.HCPhysVmcs));
5383 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5384 pCache->TestOut.pCache));
5385 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5386 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5387 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5388 pCache->TestOut.pCtx));
5389 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5390#endif
5391 NOREF(pCtx);
5392 return rc;
5393}
5394
5395
5396/**
5397 * Initialize the VMCS-Read cache.
5398 *
5399 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5400 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5401 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5402 * (those that have a 32-bit FULL & HIGH part).
5403 *
5404 * @returns VBox status code.
5405 * @param pVCpu The cross context virtual CPU structure.
5406 */
5407static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5408{
5409#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5410 do { \
5411 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5412 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5413 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5414 ++cReadFields; \
5415 } while (0)
5416
5417 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5418 uint32_t cReadFields = 0;
5419
5420 /*
5421 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5422 * and serve to indicate exceptions to the rules.
5423 */
5424
5425 /* Guest-natural selector base fields. */
5426#if 0
5427 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5430#endif
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5438 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5441 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5443#if 0
5444 /* Unused natural width guest-state fields. */
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5447#endif
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5450
5451 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5452 these 64-bit fields (using "FULL" and "HIGH" fields). */
5453#if 0
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5463#endif
5464
5465 /* Natural width guest-state fields. */
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5467#if 0
5468 /* Currently unused field. */
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5470#endif
5471
5472 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5473 {
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5475 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5476 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5477 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5478 }
5479 else
5480 {
5481 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5482 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5483 }
5484
5485#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5486 return VINF_SUCCESS;
5487}
5488
5489
5490/**
5491 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5492 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5493 * darwin, running 64-bit guests).
5494 *
5495 * @returns VBox status code.
5496 * @param pVCpu The cross context virtual CPU structure.
5497 * @param idxField The VMCS field encoding.
5498 * @param u64Val 16, 32 or 64-bit value.
5499 */
5500VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5501{
5502 int rc;
5503 switch (idxField)
5504 {
5505 /*
5506 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5507 */
5508 /* 64-bit Control fields. */
5509 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5510 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5511 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5512 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5513 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5514 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5515 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5516 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5517 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5518 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5519 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5520 case VMX_VMCS64_CTRL_EPTP_FULL:
5521 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5522 /* 64-bit Guest-state fields. */
5523 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5524 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5525 case VMX_VMCS64_GUEST_PAT_FULL:
5526 case VMX_VMCS64_GUEST_EFER_FULL:
5527 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5528 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5529 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5530 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5531 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5532 /* 64-bit Host-state fields. */
5533 case VMX_VMCS64_HOST_PAT_FULL:
5534 case VMX_VMCS64_HOST_EFER_FULL:
5535 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5536 {
5537 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5538 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5539 break;
5540 }
5541
5542 /*
5543 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5544 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5545 */
5546 /* Natural-width Guest-state fields. */
5547 case VMX_VMCS_GUEST_CR3:
5548 case VMX_VMCS_GUEST_ES_BASE:
5549 case VMX_VMCS_GUEST_CS_BASE:
5550 case VMX_VMCS_GUEST_SS_BASE:
5551 case VMX_VMCS_GUEST_DS_BASE:
5552 case VMX_VMCS_GUEST_FS_BASE:
5553 case VMX_VMCS_GUEST_GS_BASE:
5554 case VMX_VMCS_GUEST_LDTR_BASE:
5555 case VMX_VMCS_GUEST_TR_BASE:
5556 case VMX_VMCS_GUEST_GDTR_BASE:
5557 case VMX_VMCS_GUEST_IDTR_BASE:
5558 case VMX_VMCS_GUEST_RSP:
5559 case VMX_VMCS_GUEST_RIP:
5560 case VMX_VMCS_GUEST_SYSENTER_ESP:
5561 case VMX_VMCS_GUEST_SYSENTER_EIP:
5562 {
5563 if (!(RT_HI_U32(u64Val)))
5564 {
5565 /* If this field is 64-bit, VT-x will zero out the top bits. */
5566 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5567 }
5568 else
5569 {
5570 /* Assert that only the 32->64 switcher case should ever come here. */
5571 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5572 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5573 }
5574 break;
5575 }
5576
5577 default:
5578 {
5579 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5580 rc = VERR_INVALID_PARAMETER;
5581 break;
5582 }
5583 }
5584 AssertRCReturn(rc, rc);
5585 return rc;
5586}
5587
5588
5589/**
5590 * Queue up a VMWRITE by using the VMCS write cache.
5591 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5592 *
5593 * @param pVCpu The cross context virtual CPU structure.
5594 * @param idxField The VMCS field encoding.
5595 * @param u64Val 16, 32 or 64-bit value.
5596 */
5597VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5598{
5599 AssertPtr(pVCpu);
5600 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5601
5602 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5603 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5604
5605 /* Make sure there are no duplicates. */
5606 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5607 {
5608 if (pCache->Write.aField[i] == idxField)
5609 {
5610 pCache->Write.aFieldVal[i] = u64Val;
5611 return VINF_SUCCESS;
5612 }
5613 }
5614
5615 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5616 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5617 pCache->Write.cValidEntries++;
5618 return VINF_SUCCESS;
5619}
5620#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5621
5622
5623/**
5624 * Sets up the usage of TSC-offsetting and updates the VMCS.
5625 *
5626 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5627 * VMX preemption timer.
5628 *
5629 * @returns VBox status code.
5630 * @param pVCpu The cross context virtual CPU structure.
5631 *
5632 * @remarks No-long-jump zone!!!
5633 */
5634static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5635{
5636 bool fOffsettedTsc;
5637 bool fParavirtTsc;
5638 PVM pVM = pVCpu->CTX_SUFF(pVM);
5639 uint64_t uTscOffset;
5640 if (pVM->hm.s.vmx.fUsePreemptTimer)
5641 {
5642 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5643
5644 /* Make sure the returned values have sane upper and lower boundaries. */
5645 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5646 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5647 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5648 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5649
5650 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5651 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5652 AssertRC(rc);
5653 }
5654 else
5655 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5656
5657 /** @todo later optimize this to be done elsewhere and not before every
5658 * VM-entry. */
5659 if (fParavirtTsc)
5660 {
5661 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5662 information before every VM-entry, hence disable it for performance sake. */
5663#if 0
5664 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5665 AssertRC(rc);
5666#endif
5667 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5668 }
5669
5670 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5671 if ( fOffsettedTsc
5672 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5673 {
5674 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5675 {
5676 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5677 AssertRC(rc);
5678 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5679 }
5680
5681 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5682 {
5683 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5684 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5685 AssertRC(rc);
5686 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5687 }
5688 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5689 }
5690 else
5691 {
5692 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5693 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5694 {
5695 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5696 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5697 AssertRC(rc);
5698 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5699 }
5700 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5701 }
5702}
5703
5704
5705/**
5706 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5707 * VM-exit interruption info type.
5708 *
5709 * @returns The IEM exception flags.
5710 * @param uVector The event vector.
5711 * @param uVmxVectorType The VMX event type.
5712 *
5713 * @remarks This function currently only constructs flags required for
5714 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5715 * and CR2 aspects of an exception are not included).
5716 */
5717static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5718{
5719 uint32_t fIemXcptFlags;
5720 switch (uVmxVectorType)
5721 {
5722 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5723 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5724 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5725 break;
5726
5727 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5728 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5729 break;
5730
5731 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5732 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5733 break;
5734
5735 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5736 {
5737 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5738 if (uVector == X86_XCPT_BP)
5739 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5740 else if (uVector == X86_XCPT_OF)
5741 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5742 else
5743 {
5744 fIemXcptFlags = 0;
5745 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5746 }
5747 break;
5748 }
5749
5750 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5751 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5752 break;
5753
5754 default:
5755 fIemXcptFlags = 0;
5756 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5757 break;
5758 }
5759 return fIemXcptFlags;
5760}
5761
5762
5763/**
5764 * Sets an event as a pending event to be injected into the guest.
5765 *
5766 * @param pVCpu The cross context virtual CPU structure.
5767 * @param u32IntInfo The VM-entry interruption-information field.
5768 * @param cbInstr The VM-entry instruction length in bytes (for software
5769 * interrupts, exceptions and privileged software
5770 * exceptions).
5771 * @param u32ErrCode The VM-entry exception error code.
5772 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5773 * page-fault.
5774 *
5775 * @remarks Statistics counter assumes this is a guest event being injected or
5776 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5777 * always incremented.
5778 */
5779DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5780 RTGCUINTPTR GCPtrFaultAddress)
5781{
5782 Assert(!pVCpu->hm.s.Event.fPending);
5783 pVCpu->hm.s.Event.fPending = true;
5784 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5785 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5786 pVCpu->hm.s.Event.cbInstr = cbInstr;
5787 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5788}
5789
5790
5791/**
5792 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5793 *
5794 * @param pVCpu The cross context virtual CPU structure.
5795 */
5796DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5797{
5798 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5799 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5800 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5801 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5802}
5803
5804
5805/**
5806 * Handle a condition that occurred while delivering an event through the guest
5807 * IDT.
5808 *
5809 * @returns Strict VBox status code (i.e. informational status codes too).
5810 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5811 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5812 * to continue execution of the guest which will delivery the \#DF.
5813 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5814 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5815 *
5816 * @param pVCpu The cross context virtual CPU structure.
5817 * @param pVmxTransient Pointer to the VMX transient structure.
5818 *
5819 * @remarks No-long-jump zone!!!
5820 */
5821static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5822{
5823 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5824
5825 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5826 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5827 AssertRCReturn(rc2, rc2);
5828
5829 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5830 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5831 {
5832 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5833 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5834
5835 /*
5836 * If the event was a software interrupt (generated with INT n) or a software exception
5837 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5838 * can handle the VM-exit and continue guest execution which will re-execute the
5839 * instruction rather than re-injecting the exception, as that can cause premature
5840 * trips to ring-3 before injection and involve TRPM which currently has no way of
5841 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5842 * the problem).
5843 */
5844 IEMXCPTRAISE enmRaise;
5845 IEMXCPTRAISEINFO fRaiseInfo;
5846 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5847 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5848 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5849 {
5850 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5851 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5852 }
5853 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5854 {
5855 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5856 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5857 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5858 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5859 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5860 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5861 uExitVectorType), VERR_VMX_IPE_5);
5862
5863 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5864
5865 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5866 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5867 {
5868 pVmxTransient->fVectoringPF = true;
5869 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5870 }
5871 }
5872 else
5873 {
5874 /*
5875 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5876 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5877 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5878 */
5879 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5880 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5881 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5882 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5883 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5884 }
5885
5886 /*
5887 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5888 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5889 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5890 * subsequent VM-entry would fail.
5891 *
5892 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5893 */
5894 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5895 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5896 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5897 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5898 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5899 {
5900 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5901 }
5902
5903 switch (enmRaise)
5904 {
5905 case IEMXCPTRAISE_CURRENT_XCPT:
5906 {
5907 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5908 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5909 Assert(rcStrict == VINF_SUCCESS);
5910 break;
5911 }
5912
5913 case IEMXCPTRAISE_PREV_EVENT:
5914 {
5915 uint32_t u32ErrCode;
5916 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5917 {
5918 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5919 AssertRCReturn(rc2, rc2);
5920 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5921 }
5922 else
5923 u32ErrCode = 0;
5924
5925 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5926 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5927 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5928 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5929
5930 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5931 pVCpu->hm.s.Event.u32ErrCode));
5932 Assert(rcStrict == VINF_SUCCESS);
5933 break;
5934 }
5935
5936 case IEMXCPTRAISE_REEXEC_INSTR:
5937 Assert(rcStrict == VINF_SUCCESS);
5938 break;
5939
5940 case IEMXCPTRAISE_DOUBLE_FAULT:
5941 {
5942 /*
5943 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5944 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5945 */
5946 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5947 {
5948 pVmxTransient->fVectoringDoublePF = true;
5949 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5950 pVCpu->cpum.GstCtx.cr2));
5951 rcStrict = VINF_SUCCESS;
5952 }
5953 else
5954 {
5955 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5956 hmR0VmxSetPendingXcptDF(pVCpu);
5957 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5958 uIdtVector, uExitVector));
5959 rcStrict = VINF_HM_DOUBLE_FAULT;
5960 }
5961 break;
5962 }
5963
5964 case IEMXCPTRAISE_TRIPLE_FAULT:
5965 {
5966 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5967 rcStrict = VINF_EM_RESET;
5968 break;
5969 }
5970
5971 case IEMXCPTRAISE_CPU_HANG:
5972 {
5973 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5974 rcStrict = VERR_EM_GUEST_CPU_HANG;
5975 break;
5976 }
5977
5978 default:
5979 {
5980 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5981 rcStrict = VERR_VMX_IPE_2;
5982 break;
5983 }
5984 }
5985 }
5986 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5987 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5988 && uExitVector != X86_XCPT_DF
5989 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5990 {
5991 /*
5992 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5993 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5994 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5995 */
5996 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5997 {
5998 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5999 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6000 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6001 }
6002 }
6003
6004 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6005 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6006 return rcStrict;
6007}
6008
6009
6010/**
6011 * Imports a guest segment register from the current VMCS into
6012 * the guest-CPU context.
6013 *
6014 * @returns VBox status code.
6015 * @param pVCpu The cross context virtual CPU structure.
6016 * @param idxSel Index of the selector in the VMCS.
6017 * @param idxLimit Index of the segment limit in the VMCS.
6018 * @param idxBase Index of the segment base in the VMCS.
6019 * @param idxAccess Index of the access rights of the segment in the VMCS.
6020 * @param pSelReg Pointer to the segment selector.
6021 *
6022 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6023 * do not log!
6024 *
6025 * @remarks Never call this function directly!!! Use the
6026 * HMVMX_IMPORT_SREG() macro as that takes care
6027 * of whether to read from the VMCS cache or not.
6028 */
6029static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6030 PCPUMSELREG pSelReg)
6031{
6032 NOREF(pVCpu);
6033
6034 uint32_t u32Sel;
6035 uint32_t u32Limit;
6036 uint32_t u32Attr;
6037 uint64_t u64Base;
6038 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6039 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6040 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6041 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6042 AssertRCReturn(rc, rc);
6043
6044 pSelReg->Sel = (uint16_t)u32Sel;
6045 pSelReg->ValidSel = (uint16_t)u32Sel;
6046 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6047 pSelReg->u32Limit = u32Limit;
6048 pSelReg->u64Base = u64Base;
6049 pSelReg->Attr.u = u32Attr;
6050
6051 /*
6052 * If VT-x marks the segment as unusable, most other bits remain undefined:
6053 * - For CS the L, D and G bits have meaning.
6054 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6055 * - For the remaining data segments no bits are defined.
6056 *
6057 * The present bit and the unusable bit has been observed to be set at the
6058 * same time (the selector was supposed to be invalid as we started executing
6059 * a V8086 interrupt in ring-0).
6060 *
6061 * What should be important for the rest of the VBox code, is that the P bit is
6062 * cleared. Some of the other VBox code recognizes the unusable bit, but
6063 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6064 * safe side here, we'll strip off P and other bits we don't care about. If
6065 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6066 *
6067 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6068 */
6069 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6070 {
6071 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6072
6073 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6074 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6075 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6076#ifdef VBOX_STRICT
6077 VMMRZCallRing3Disable(pVCpu);
6078 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6079# ifdef DEBUG_bird
6080 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6081 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6082 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6083# endif
6084 VMMRZCallRing3Enable(pVCpu);
6085#endif
6086 }
6087 return VINF_SUCCESS;
6088}
6089
6090
6091/**
6092 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6093 *
6094 * @returns VBox status code.
6095 * @param pVCpu The cross context virtual CPU structure.
6096 *
6097 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6098 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6099 * instead!!!
6100 */
6101DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6102{
6103 uint64_t u64Val;
6104 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6105 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6106 {
6107 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6108 if (RT_SUCCESS(rc))
6109 {
6110 pCtx->rip = u64Val;
6111 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6112 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6113 }
6114 return rc;
6115 }
6116 return VINF_SUCCESS;
6117}
6118
6119
6120/**
6121 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6122 *
6123 * @returns VBox status code.
6124 * @param pVCpu The cross context virtual CPU structure.
6125 *
6126 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6127 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6128 * instead!!!
6129 */
6130DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6131{
6132 uint32_t u32Val;
6133 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6134 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6135 {
6136 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6137 if (RT_SUCCESS(rc))
6138 {
6139 pCtx->eflags.u32 = u32Val;
6140
6141 /* Restore eflags for real-on-v86-mode hack. */
6142 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6143 {
6144 pCtx->eflags.Bits.u1VM = 0;
6145 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6146 }
6147 }
6148 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6149 return rc;
6150 }
6151 return VINF_SUCCESS;
6152}
6153
6154
6155/**
6156 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6157 * context.
6158 *
6159 * @returns VBox status code.
6160 * @param pVCpu The cross context virtual CPU structure.
6161 *
6162 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6163 * do not log!
6164 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6165 * instead!!!
6166 */
6167DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6168{
6169 uint32_t u32Val;
6170 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6171 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6172 if (RT_SUCCESS(rc))
6173 {
6174 /*
6175 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6176 * might need them in hmR0VmxEvaluatePendingEvent().
6177 */
6178 if (!u32Val)
6179 {
6180 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6181 {
6182 rc = hmR0VmxImportGuestRip(pVCpu);
6183 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6184 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6185 }
6186
6187 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6188 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6189 }
6190 else
6191 {
6192 rc = hmR0VmxImportGuestRip(pVCpu);
6193 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6194
6195 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6196 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6197 {
6198 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6199 }
6200 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6202
6203 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6204 {
6205 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6206 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6207 }
6208 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6209 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6210 }
6211 }
6212 return rc;
6213}
6214
6215
6216/**
6217 * Worker for VMXR0ImportStateOnDemand.
6218 *
6219 * @returns VBox status code.
6220 * @param pVCpu The cross context virtual CPU structure.
6221 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6222 */
6223static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6224{
6225#define VMXLOCAL_BREAK_RC(a_rc) \
6226 if (RT_FAILURE(a_rc)) \
6227 break
6228
6229 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6230
6231 int rc = VINF_SUCCESS;
6232 PVM pVM = pVCpu->CTX_SUFF(pVM);
6233 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6234 uint64_t u64Val;
6235 uint32_t u32Val;
6236
6237 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6238
6239 /*
6240 * We disable interrupts to make the updating of the state and in particular
6241 * the fExtrn modification atomic wrt to preemption hooks.
6242 */
6243 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6244
6245 fWhat &= pCtx->fExtrn;
6246 if (fWhat)
6247 {
6248 do
6249 {
6250 if (fWhat & CPUMCTX_EXTRN_RIP)
6251 {
6252 rc = hmR0VmxImportGuestRip(pVCpu);
6253 VMXLOCAL_BREAK_RC(rc);
6254 }
6255
6256 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6257 {
6258 rc = hmR0VmxImportGuestRFlags(pVCpu);
6259 VMXLOCAL_BREAK_RC(rc);
6260 }
6261
6262 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6263 {
6264 rc = hmR0VmxImportGuestIntrState(pVCpu);
6265 VMXLOCAL_BREAK_RC(rc);
6266 }
6267
6268 if (fWhat & CPUMCTX_EXTRN_RSP)
6269 {
6270 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6271 VMXLOCAL_BREAK_RC(rc);
6272 pCtx->rsp = u64Val;
6273 }
6274
6275 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6276 {
6277 if (fWhat & CPUMCTX_EXTRN_CS)
6278 {
6279 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6280 rc |= hmR0VmxImportGuestRip(pVCpu);
6281 VMXLOCAL_BREAK_RC(rc);
6282 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6283 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6284 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6285 }
6286 if (fWhat & CPUMCTX_EXTRN_SS)
6287 {
6288 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6289 VMXLOCAL_BREAK_RC(rc);
6290 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6291 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6292 }
6293 if (fWhat & CPUMCTX_EXTRN_DS)
6294 {
6295 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6296 VMXLOCAL_BREAK_RC(rc);
6297 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6298 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6299 }
6300 if (fWhat & CPUMCTX_EXTRN_ES)
6301 {
6302 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6303 VMXLOCAL_BREAK_RC(rc);
6304 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6305 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6306 }
6307 if (fWhat & CPUMCTX_EXTRN_FS)
6308 {
6309 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6310 VMXLOCAL_BREAK_RC(rc);
6311 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6312 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6313 }
6314 if (fWhat & CPUMCTX_EXTRN_GS)
6315 {
6316 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6317 VMXLOCAL_BREAK_RC(rc);
6318 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6319 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6320 }
6321 }
6322
6323 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6324 {
6325 if (fWhat & CPUMCTX_EXTRN_LDTR)
6326 {
6327 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6328 VMXLOCAL_BREAK_RC(rc);
6329 }
6330
6331 if (fWhat & CPUMCTX_EXTRN_GDTR)
6332 {
6333 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6334 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6335 VMXLOCAL_BREAK_RC(rc);
6336 pCtx->gdtr.pGdt = u64Val;
6337 pCtx->gdtr.cbGdt = u32Val;
6338 }
6339
6340 /* Guest IDTR. */
6341 if (fWhat & CPUMCTX_EXTRN_IDTR)
6342 {
6343 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6344 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6345 VMXLOCAL_BREAK_RC(rc);
6346 pCtx->idtr.pIdt = u64Val;
6347 pCtx->idtr.cbIdt = u32Val;
6348 }
6349
6350 /* Guest TR. */
6351 if (fWhat & CPUMCTX_EXTRN_TR)
6352 {
6353 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6354 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6355 {
6356 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6357 VMXLOCAL_BREAK_RC(rc);
6358 }
6359 }
6360 }
6361
6362 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6363 {
6364 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6365 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6366 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6367 pCtx->SysEnter.cs = u32Val;
6368 VMXLOCAL_BREAK_RC(rc);
6369 }
6370
6371#if HC_ARCH_BITS == 64
6372 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6373 {
6374 if ( pVM->hm.s.fAllow64BitGuests
6375 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6376 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6377 }
6378
6379 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6380 {
6381 if ( pVM->hm.s.fAllow64BitGuests
6382 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6383 {
6384 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6385 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6386 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6387 }
6388 }
6389#endif
6390
6391 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6392#if HC_ARCH_BITS == 32
6393 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6394#endif
6395 )
6396 {
6397 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6398 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6399 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6400 {
6401 switch (pMsr->u32Msr)
6402 {
6403#if HC_ARCH_BITS == 32
6404 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6405 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6406 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6407 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6408#endif
6409 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6410 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6411 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6412 default:
6413 {
6414 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6415 ASMSetFlags(fEFlags);
6416 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6417 cMsrs));
6418 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6419 }
6420 }
6421 }
6422 }
6423
6424 if (fWhat & CPUMCTX_EXTRN_DR7)
6425 {
6426 if (!pVCpu->hm.s.fUsingHyperDR7)
6427 {
6428 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6429 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6430 VMXLOCAL_BREAK_RC(rc);
6431 pCtx->dr[7] = u32Val;
6432 }
6433 }
6434
6435 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6436 {
6437 uint32_t u32Shadow;
6438 if (fWhat & CPUMCTX_EXTRN_CR0)
6439 {
6440 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6441 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6442 VMXLOCAL_BREAK_RC(rc);
6443 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6444 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6445 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6446 CPUMSetGuestCR0(pVCpu, u32Val);
6447 VMMRZCallRing3Enable(pVCpu);
6448 }
6449
6450 if (fWhat & CPUMCTX_EXTRN_CR4)
6451 {
6452 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6453 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6454 VMXLOCAL_BREAK_RC(rc);
6455 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6456 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6457 CPUMSetGuestCR4(pVCpu, u32Val);
6458 }
6459
6460 if (fWhat & CPUMCTX_EXTRN_CR3)
6461 {
6462 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6463 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6464 || ( pVM->hm.s.fNestedPaging
6465 && CPUMIsGuestPagingEnabledEx(pCtx)))
6466 {
6467 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6468 if (pCtx->cr3 != u64Val)
6469 {
6470 CPUMSetGuestCR3(pVCpu, u64Val);
6471 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6472 }
6473
6474 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6475 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6476 if (CPUMIsGuestInPAEModeEx(pCtx))
6477 {
6478 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6479 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6480 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6481 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6482 VMXLOCAL_BREAK_RC(rc);
6483 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6484 }
6485 }
6486 }
6487 }
6488 } while (0);
6489
6490 if (RT_SUCCESS(rc))
6491 {
6492 /* Update fExtrn. */
6493 pCtx->fExtrn &= ~fWhat;
6494
6495 /* If everything has been imported, clear the HM keeper bit. */
6496 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6497 {
6498 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6499 Assert(!pCtx->fExtrn);
6500 }
6501 }
6502 }
6503 else
6504 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6505
6506 ASMSetFlags(fEFlags);
6507
6508 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6509
6510 /*
6511 * Honor any pending CR3 updates.
6512 *
6513 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6514 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6515 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6516 *
6517 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6518 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6519 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6520 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6521 *
6522 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6523 */
6524 if (VMMRZCallRing3IsEnabled(pVCpu))
6525 {
6526 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6527 {
6528 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6529 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6530 }
6531
6532 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6533 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6534
6535 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6536 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6537 }
6538
6539 return VINF_SUCCESS;
6540#undef VMXLOCAL_BREAK_RC
6541}
6542
6543
6544/**
6545 * Saves the guest state from the VMCS into the guest-CPU context.
6546 *
6547 * @returns VBox status code.
6548 * @param pVCpu The cross context virtual CPU structure.
6549 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6550 */
6551VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6552{
6553 return hmR0VmxImportGuestState(pVCpu, fWhat);
6554}
6555
6556
6557/**
6558 * Check per-VM and per-VCPU force flag actions that require us to go back to
6559 * ring-3 for one reason or another.
6560 *
6561 * @returns Strict VBox status code (i.e. informational status codes too)
6562 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6563 * ring-3.
6564 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6565 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6566 * interrupts)
6567 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6568 * all EMTs to be in ring-3.
6569 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6570 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6571 * to the EM loop.
6572 *
6573 * @param pVCpu The cross context virtual CPU structure.
6574 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6575 */
6576static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6577{
6578 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6579
6580 /*
6581 * Anything pending? Should be more likely than not if we're doing a good job.
6582 */
6583 PVM pVM = pVCpu->CTX_SUFF(pVM);
6584 if ( !fStepping
6585 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6586 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6587 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6588 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6589 return VINF_SUCCESS;
6590
6591 /* Pending PGM C3 sync. */
6592 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6593 {
6594 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6595 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6596 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6597 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6598 if (rcStrict2 != VINF_SUCCESS)
6599 {
6600 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6601 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6602 return rcStrict2;
6603 }
6604 }
6605
6606 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6607 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6608 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6609 {
6610 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6611 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6612 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6613 return rc2;
6614 }
6615
6616 /* Pending VM request packets, such as hardware interrupts. */
6617 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6618 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6619 {
6620 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6621 return VINF_EM_PENDING_REQUEST;
6622 }
6623
6624 /* Pending PGM pool flushes. */
6625 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6626 {
6627 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6628 return VINF_PGM_POOL_FLUSH_PENDING;
6629 }
6630
6631 /* Pending DMA requests. */
6632 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6633 {
6634 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6635 return VINF_EM_RAW_TO_R3;
6636 }
6637
6638 return VINF_SUCCESS;
6639}
6640
6641
6642/**
6643 * Converts any TRPM trap into a pending HM event. This is typically used when
6644 * entering from ring-3 (not longjmp returns).
6645 *
6646 * @param pVCpu The cross context virtual CPU structure.
6647 */
6648static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6649{
6650 Assert(TRPMHasTrap(pVCpu));
6651 Assert(!pVCpu->hm.s.Event.fPending);
6652
6653 uint8_t uVector;
6654 TRPMEVENT enmTrpmEvent;
6655 RTGCUINT uErrCode;
6656 RTGCUINTPTR GCPtrFaultAddress;
6657 uint8_t cbInstr;
6658
6659 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6660 AssertRC(rc);
6661
6662 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6663 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6664 if (enmTrpmEvent == TRPM_TRAP)
6665 {
6666 switch (uVector)
6667 {
6668 case X86_XCPT_NMI:
6669 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6670 break;
6671
6672 case X86_XCPT_BP:
6673 case X86_XCPT_OF:
6674 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6675 break;
6676
6677 case X86_XCPT_PF:
6678 case X86_XCPT_DF:
6679 case X86_XCPT_TS:
6680 case X86_XCPT_NP:
6681 case X86_XCPT_SS:
6682 case X86_XCPT_GP:
6683 case X86_XCPT_AC:
6684 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6685 RT_FALL_THRU();
6686 default:
6687 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6688 break;
6689 }
6690 }
6691 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6692 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6693 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6694 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6695 else
6696 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6697
6698 rc = TRPMResetTrap(pVCpu);
6699 AssertRC(rc);
6700 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6701 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6702
6703 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6704}
6705
6706
6707/**
6708 * Converts the pending HM event into a TRPM trap.
6709 *
6710 * @param pVCpu The cross context virtual CPU structure.
6711 */
6712static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6713{
6714 Assert(pVCpu->hm.s.Event.fPending);
6715
6716 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6717 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6718 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6719 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6720
6721 /* If a trap was already pending, we did something wrong! */
6722 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6723
6724 TRPMEVENT enmTrapType;
6725 switch (uVectorType)
6726 {
6727 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6728 enmTrapType = TRPM_HARDWARE_INT;
6729 break;
6730
6731 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6732 enmTrapType = TRPM_SOFTWARE_INT;
6733 break;
6734
6735 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6736 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6737 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6738 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6739 enmTrapType = TRPM_TRAP;
6740 break;
6741
6742 default:
6743 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6744 enmTrapType = TRPM_32BIT_HACK;
6745 break;
6746 }
6747
6748 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6749
6750 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6751 AssertRC(rc);
6752
6753 if (fErrorCodeValid)
6754 TRPMSetErrorCode(pVCpu, uErrorCode);
6755
6756 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6757 && uVector == X86_XCPT_PF)
6758 {
6759 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6760 }
6761 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6762 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6763 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6764 {
6765 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6766 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6767 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6768 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6769 }
6770
6771 /* Clear any pending events from the VMCS. */
6772 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6773 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6774
6775 /* We're now done converting the pending event. */
6776 pVCpu->hm.s.Event.fPending = false;
6777}
6778
6779
6780/**
6781 * Does the necessary state syncing before returning to ring-3 for any reason
6782 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6783 *
6784 * @returns VBox status code.
6785 * @param pVCpu The cross context virtual CPU structure.
6786 * @param fImportState Whether to import the guest state from the VMCS back
6787 * to the guest-CPU context.
6788 *
6789 * @remarks No-long-jmp zone!!!
6790 */
6791static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6792{
6793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6794 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6795
6796 RTCPUID idCpu = RTMpCpuId();
6797 Log4Func(("HostCpuId=%u\n", idCpu));
6798
6799 /*
6800 * !!! IMPORTANT !!!
6801 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6802 */
6803
6804 /* Save the guest state if necessary. */
6805 if (fImportState)
6806 {
6807 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6808 AssertRCReturn(rc, rc);
6809 }
6810
6811 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6812 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6813 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6814
6815 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6816#ifdef VBOX_STRICT
6817 if (CPUMIsHyperDebugStateActive(pVCpu))
6818 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6819#endif
6820 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6821 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6822 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6823
6824#if HC_ARCH_BITS == 64
6825 /* Restore host-state bits that VT-x only restores partially. */
6826 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6827 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6828 {
6829 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6830 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6831 }
6832 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6833#endif
6834
6835 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6836 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6837 {
6838 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6839 if (!fImportState)
6840 {
6841 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6842 AssertRCReturn(rc, rc);
6843 }
6844 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6845 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6846 }
6847 else
6848 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6849
6850 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6851 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6852
6853 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6854 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6855 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6856 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6857 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6858 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6859 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6860 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6861 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6862
6863 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6864
6865 /** @todo This partially defeats the purpose of having preemption hooks.
6866 * The problem is, deregistering the hooks should be moved to a place that
6867 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6868 * context.
6869 */
6870 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6871 {
6872 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6873 AssertRCReturn(rc, rc);
6874
6875 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6876 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6877 }
6878 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6879 NOREF(idCpu);
6880
6881 return VINF_SUCCESS;
6882}
6883
6884
6885/**
6886 * Leaves the VT-x session.
6887 *
6888 * @returns VBox status code.
6889 * @param pVCpu The cross context virtual CPU structure.
6890 *
6891 * @remarks No-long-jmp zone!!!
6892 */
6893static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6894{
6895 HM_DISABLE_PREEMPT();
6896 HMVMX_ASSERT_CPU_SAFE();
6897 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6898 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6899
6900 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6901 and done this from the VMXR0ThreadCtxCallback(). */
6902 if (!pVCpu->hm.s.fLeaveDone)
6903 {
6904 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6905 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6906 pVCpu->hm.s.fLeaveDone = true;
6907 }
6908 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6909
6910 /*
6911 * !!! IMPORTANT !!!
6912 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6913 */
6914
6915 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6916 /** @todo Deregistering here means we need to VMCLEAR always
6917 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6918 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6919 VMMR0ThreadCtxHookDisable(pVCpu);
6920
6921 /* Leave HM context. This takes care of local init (term). */
6922 int rc = HMR0LeaveCpu(pVCpu);
6923
6924 HM_RESTORE_PREEMPT();
6925 return rc;
6926}
6927
6928
6929/**
6930 * Does the necessary state syncing before doing a longjmp to ring-3.
6931 *
6932 * @returns VBox status code.
6933 * @param pVCpu The cross context virtual CPU structure.
6934 *
6935 * @remarks No-long-jmp zone!!!
6936 */
6937DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6938{
6939 return hmR0VmxLeaveSession(pVCpu);
6940}
6941
6942
6943/**
6944 * Take necessary actions before going back to ring-3.
6945 *
6946 * An action requires us to go back to ring-3. This function does the necessary
6947 * steps before we can safely return to ring-3. This is not the same as longjmps
6948 * to ring-3, this is voluntary and prepares the guest so it may continue
6949 * executing outside HM (recompiler/IEM).
6950 *
6951 * @returns VBox status code.
6952 * @param pVCpu The cross context virtual CPU structure.
6953 * @param rcExit The reason for exiting to ring-3. Can be
6954 * VINF_VMM_UNKNOWN_RING3_CALL.
6955 */
6956static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6957{
6958 Assert(pVCpu);
6959 HMVMX_ASSERT_PREEMPT_SAFE();
6960
6961 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6962 {
6963 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6964 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6965 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6966 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6967 }
6968
6969 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6970 VMMRZCallRing3Disable(pVCpu);
6971 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6972
6973 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6974 if (pVCpu->hm.s.Event.fPending)
6975 {
6976 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6977 Assert(!pVCpu->hm.s.Event.fPending);
6978 }
6979
6980 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6981 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6982
6983 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6984 and if we're injecting an event we should have a TRPM trap pending. */
6985 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6986#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6987 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6988#endif
6989
6990 /* Save guest state and restore host state bits. */
6991 int rc = hmR0VmxLeaveSession(pVCpu);
6992 AssertRCReturn(rc, rc);
6993 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6994 /* Thread-context hooks are unregistered at this point!!! */
6995
6996 /* Sync recompiler state. */
6997 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6998 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6999 | CPUM_CHANGED_LDTR
7000 | CPUM_CHANGED_GDTR
7001 | CPUM_CHANGED_IDTR
7002 | CPUM_CHANGED_TR
7003 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7004 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7005 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7006 {
7007 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7008 }
7009
7010 Assert(!pVCpu->hm.s.fClearTrapFlag);
7011
7012 /* Update the exit-to-ring 3 reason. */
7013 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7014
7015 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7016 if (rcExit != VINF_EM_RAW_INTERRUPT)
7017 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7018
7019 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7020
7021 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7022 VMMRZCallRing3RemoveNotification(pVCpu);
7023 VMMRZCallRing3Enable(pVCpu);
7024
7025 return rc;
7026}
7027
7028
7029/**
7030 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7031 * longjump to ring-3 and possibly get preempted.
7032 *
7033 * @returns VBox status code.
7034 * @param pVCpu The cross context virtual CPU structure.
7035 * @param enmOperation The operation causing the ring-3 longjump.
7036 * @param pvUser User argument, currently unused, NULL.
7037 */
7038static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7039{
7040 RT_NOREF(pvUser);
7041 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7042 {
7043 /*
7044 * !!! IMPORTANT !!!
7045 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7046 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7047 */
7048 VMMRZCallRing3RemoveNotification(pVCpu);
7049 VMMRZCallRing3Disable(pVCpu);
7050 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7051 RTThreadPreemptDisable(&PreemptState);
7052
7053 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7054 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7055 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7056
7057#if HC_ARCH_BITS == 64
7058 /* Restore host-state bits that VT-x only restores partially. */
7059 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7060 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7061 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7062 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7063#endif
7064
7065 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7066 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7067 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7068
7069 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7070 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7071 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7072 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7073 {
7074 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7075 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7076 }
7077
7078 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7079 VMMR0ThreadCtxHookDisable(pVCpu);
7080 HMR0LeaveCpu(pVCpu);
7081 RTThreadPreemptRestore(&PreemptState);
7082 return VINF_SUCCESS;
7083 }
7084
7085 Assert(pVCpu);
7086 Assert(pvUser);
7087 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7088 HMVMX_ASSERT_PREEMPT_SAFE();
7089
7090 VMMRZCallRing3Disable(pVCpu);
7091 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7092
7093 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7094
7095 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7096 AssertRCReturn(rc, rc);
7097
7098 VMMRZCallRing3Enable(pVCpu);
7099 return VINF_SUCCESS;
7100}
7101
7102
7103/**
7104 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7105 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7106 *
7107 * @param pVCpu The cross context virtual CPU structure.
7108 */
7109DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7110{
7111 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7112 {
7113 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7114 {
7115 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7116 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7117 AssertRC(rc);
7118 Log4Func(("Setup interrupt-window exiting\n"));
7119 }
7120 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7121}
7122
7123
7124/**
7125 * Clears the interrupt-window exiting control in the VMCS.
7126 *
7127 * @param pVCpu The cross context virtual CPU structure.
7128 */
7129DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7130{
7131 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7132 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7133 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7134 AssertRC(rc);
7135 Log4Func(("Cleared interrupt-window exiting\n"));
7136}
7137
7138
7139/**
7140 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7141 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7142 *
7143 * @param pVCpu The cross context virtual CPU structure.
7144 */
7145DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7146{
7147 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7148 {
7149 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7150 {
7151 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7152 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7153 AssertRC(rc);
7154 Log4Func(("Setup NMI-window exiting\n"));
7155 }
7156 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7157}
7158
7159
7160/**
7161 * Clears the NMI-window exiting control in the VMCS.
7162 *
7163 * @param pVCpu The cross context virtual CPU structure.
7164 */
7165DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7166{
7167 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7168 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7169 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7170 AssertRC(rc);
7171 Log4Func(("Cleared NMI-window exiting\n"));
7172}
7173
7174
7175/**
7176 * Evaluates the event to be delivered to the guest and sets it as the pending
7177 * event.
7178 *
7179 * @returns The VT-x guest-interruptibility state.
7180 * @param pVCpu The cross context virtual CPU structure.
7181 */
7182static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7183{
7184 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7185 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7186 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7187 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7188 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7189 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7190
7191 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7192 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7193 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7194 Assert(!TRPMHasTrap(pVCpu));
7195
7196 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7197 APICUpdatePendingInterrupts(pVCpu);
7198
7199 /*
7200 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7201 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7202 */
7203 /** @todo SMI. SMIs take priority over NMIs. */
7204 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7205 {
7206 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7207 if ( !pVCpu->hm.s.Event.fPending
7208 && !fBlockNmi
7209 && !fBlockSti
7210 && !fBlockMovSS)
7211 {
7212 Log4Func(("Pending NMI\n"));
7213 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7214 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7215
7216 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7217 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7218 }
7219 else
7220 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7221 }
7222 /*
7223 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7224 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7225 */
7226 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7227 && !pVCpu->hm.s.fSingleInstruction)
7228 {
7229 Assert(!DBGFIsStepping(pVCpu));
7230 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7231 AssertRCReturn(rc, 0);
7232 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7233 if ( !pVCpu->hm.s.Event.fPending
7234 && !fBlockInt
7235 && !fBlockSti
7236 && !fBlockMovSS)
7237 {
7238 uint8_t u8Interrupt;
7239 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7240 if (RT_SUCCESS(rc))
7241 {
7242 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7243 uint32_t u32IntInfo = u8Interrupt
7244 | VMX_EXIT_INTERRUPTION_INFO_VALID
7245 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7246
7247 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7248 }
7249 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7250 {
7251 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7252 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7253 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7254
7255 /*
7256 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7257 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7258 * need to re-set this force-flag here.
7259 */
7260 }
7261 else
7262 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7263 }
7264 else
7265 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7266 }
7267
7268 return fIntrState;
7269}
7270
7271
7272/**
7273 * Sets a pending-debug exception to be delivered to the guest if the guest is
7274 * single-stepping in the VMCS.
7275 *
7276 * @param pVCpu The cross context virtual CPU structure.
7277 */
7278DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7279{
7280 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7281 RT_NOREF(pVCpu);
7282 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7283}
7284
7285
7286/**
7287 * Injects any pending events into the guest if the guest is in a state to
7288 * receive them.
7289 *
7290 * @returns Strict VBox status code (i.e. informational status codes too).
7291 * @param pVCpu The cross context virtual CPU structure.
7292 * @param fIntrState The VT-x guest-interruptibility state.
7293 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7294 * return VINF_EM_DBG_STEPPED if the event was
7295 * dispatched directly.
7296 */
7297static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7298{
7299 HMVMX_ASSERT_PREEMPT_SAFE();
7300 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7301
7302 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7303 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7304
7305 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7306 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7307 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7308 Assert(!TRPMHasTrap(pVCpu));
7309
7310 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7311 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7312 if (pVCpu->hm.s.Event.fPending)
7313 {
7314 /*
7315 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7316 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7317 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7318 *
7319 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7320 */
7321 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7322#ifdef VBOX_STRICT
7323 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7324 {
7325 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7326 Assert(!fBlockInt);
7327 Assert(!fBlockSti);
7328 Assert(!fBlockMovSS);
7329 }
7330 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7331 {
7332 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7333 Assert(!fBlockSti);
7334 Assert(!fBlockMovSS);
7335 Assert(!fBlockNmi);
7336 }
7337#endif
7338 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7339 uIntType));
7340 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7341 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7342 &fIntrState);
7343 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7344
7345 /* Update the interruptibility-state as it could have been changed by
7346 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7347 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7348 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7349
7350 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7351 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7352 else
7353 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7354 }
7355
7356 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7357 if ( fBlockSti
7358 || fBlockMovSS)
7359 {
7360 if (!pVCpu->hm.s.fSingleInstruction)
7361 {
7362 /*
7363 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7364 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7365 * See Intel spec. 27.3.4 "Saving Non-Register State".
7366 */
7367 Assert(!DBGFIsStepping(pVCpu));
7368 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7369 AssertRCReturn(rc, rc);
7370 if (pCtx->eflags.Bits.u1TF)
7371 {
7372 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7373 AssertRCReturn(rc2, rc2);
7374 }
7375 }
7376 else if (pCtx->eflags.Bits.u1TF)
7377 {
7378 /*
7379 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7380 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7381 */
7382 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7383 fIntrState = 0;
7384 }
7385 }
7386
7387 /*
7388 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7389 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7390 */
7391 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7392 AssertRCReturn(rc3, rc3);
7393
7394 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7395 NOREF(fBlockMovSS); NOREF(fBlockSti);
7396 return rcStrict;
7397}
7398
7399
7400/**
7401 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7402 *
7403 * @param pVCpu The cross context virtual CPU structure.
7404 */
7405DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
7406{
7407 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7408 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7409}
7410
7411
7412/**
7413 * Injects a double-fault (\#DF) exception into the VM.
7414 *
7415 * @returns Strict VBox status code (i.e. informational status codes too).
7416 * @param pVCpu The cross context virtual CPU structure.
7417 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7418 * and should return VINF_EM_DBG_STEPPED if the event
7419 * is injected directly (register modified by us, not
7420 * by hardware on VM-entry).
7421 * @param pfIntrState Pointer to the current guest interruptibility-state.
7422 * This interruptibility-state will be updated if
7423 * necessary. This cannot not be NULL.
7424 */
7425DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7426{
7427 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7428 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7429 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7430 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7431 pfIntrState);
7432}
7433
7434
7435/**
7436 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7437 *
7438 * @param pVCpu The cross context virtual CPU structure.
7439 */
7440DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
7441{
7442 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7443 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7444 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7445}
7446
7447
7448/**
7449 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7450 *
7451 * @param pVCpu The cross context virtual CPU structure.
7452 * @param cbInstr The value of RIP that is to be pushed on the guest
7453 * stack.
7454 */
7455DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
7456{
7457 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7458 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7459 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7460}
7461
7462
7463/**
7464 * Injects a general-protection (\#GP) fault into the VM.
7465 *
7466 * @returns Strict VBox status code (i.e. informational status codes too).
7467 * @param pVCpu The cross context virtual CPU structure.
7468 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7469 * mode, i.e. in real-mode it's not valid).
7470 * @param u32ErrorCode The error code associated with the \#GP.
7471 * @param fStepping Whether we're running in
7472 * hmR0VmxRunGuestCodeStep() and should return
7473 * VINF_EM_DBG_STEPPED if the event is injected
7474 * directly (register modified by us, not by
7475 * hardware on VM-entry).
7476 * @param pfIntrState Pointer to the current guest interruptibility-state.
7477 * This interruptibility-state will be updated if
7478 * necessary. This cannot not be NULL.
7479 */
7480DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7481 uint32_t *pfIntrState)
7482{
7483 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7484 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485 if (fErrorCodeValid)
7486 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7487 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7488 pfIntrState);
7489}
7490
7491
7492/**
7493 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7494 *
7495 * @param pVCpu The cross context virtual CPU structure.
7496 * @param uVector The software interrupt vector number.
7497 * @param cbInstr The value of RIP that is to be pushed on the guest
7498 * stack.
7499 */
7500DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7501{
7502 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7503 if ( uVector == X86_XCPT_BP
7504 || uVector == X86_XCPT_OF)
7505 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7506 else
7507 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7508 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7509}
7510
7511
7512/**
7513 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7514 * stack.
7515 *
7516 * @returns Strict VBox status code (i.e. informational status codes too).
7517 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7518 * @param pVCpu The cross context virtual CPU structure.
7519 * @param uValue The value to push to the guest stack.
7520 */
7521static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7522{
7523 /*
7524 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7525 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7526 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7527 */
7528 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7529 if (pCtx->sp == 1)
7530 return VINF_EM_RESET;
7531 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7532 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7533 AssertRC(rc);
7534 return rc;
7535}
7536
7537
7538/**
7539 * Injects an event into the guest upon VM-entry by updating the relevant fields
7540 * in the VM-entry area in the VMCS.
7541 *
7542 * @returns Strict VBox status code (i.e. informational status codes too).
7543 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7544 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7545 *
7546 * @param pVCpu The cross context virtual CPU structure.
7547 * @param u64IntInfo The VM-entry interruption-information field.
7548 * @param cbInstr The VM-entry instruction length in bytes (for
7549 * software interrupts, exceptions and privileged
7550 * software exceptions).
7551 * @param u32ErrCode The VM-entry exception error code.
7552 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7553 * @param pfIntrState Pointer to the current guest interruptibility-state.
7554 * This interruptibility-state will be updated if
7555 * necessary. This cannot not be NULL.
7556 * @param fStepping Whether we're running in
7557 * hmR0VmxRunGuestCodeStep() and should return
7558 * VINF_EM_DBG_STEPPED if the event is injected
7559 * directly (register modified by us, not by
7560 * hardware on VM-entry).
7561 */
7562static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7563 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7564{
7565 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7566 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7567 Assert(pfIntrState);
7568
7569 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7570 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7571 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7572 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7573
7574#ifdef VBOX_STRICT
7575 /*
7576 * Validate the error-code-valid bit for hardware exceptions.
7577 * No error codes for exceptions in real-mode.
7578 *
7579 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7580 */
7581 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7582 && !CPUMIsGuestInRealModeEx(pCtx))
7583 {
7584 switch (uVector)
7585 {
7586 case X86_XCPT_PF:
7587 case X86_XCPT_DF:
7588 case X86_XCPT_TS:
7589 case X86_XCPT_NP:
7590 case X86_XCPT_SS:
7591 case X86_XCPT_GP:
7592 case X86_XCPT_AC:
7593 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7594 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7595 RT_FALL_THRU();
7596 default:
7597 break;
7598 }
7599 }
7600#endif
7601
7602 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7603 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7604 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7605
7606 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7607
7608 /*
7609 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7610 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7611 * interrupt handler in the (real-mode) guest.
7612 *
7613 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7614 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7615 */
7616 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7617 {
7618 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7619 {
7620 /*
7621 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7622 * set the deliver-error-code bit.
7623 *
7624 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7625 */
7626 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7627 }
7628 else
7629 {
7630 PVM pVM = pVCpu->CTX_SUFF(pVM);
7631 Assert(PDMVmmDevHeapIsEnabled(pVM));
7632 Assert(pVM->hm.s.vmx.pRealModeTSS);
7633
7634 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7635 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7636 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7637 AssertRCReturn(rc2, rc2);
7638
7639 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7640 size_t const cbIdtEntry = sizeof(X86IDTR16);
7641 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7642 {
7643 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7644 if (uVector == X86_XCPT_DF)
7645 return VINF_EM_RESET;
7646
7647 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7648 if (uVector == X86_XCPT_GP)
7649 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7650
7651 /*
7652 * If we're injecting an event with no valid IDT entry, inject a #GP.
7653 * No error codes for exceptions in real-mode.
7654 *
7655 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7656 */
7657 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7658 }
7659
7660 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7661 uint16_t uGuestIp = pCtx->ip;
7662 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7663 {
7664 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7665 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7666 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7667 }
7668 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7669 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7670
7671 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7672 X86IDTR16 IdtEntry;
7673 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7674 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7675 AssertRCReturn(rc2, rc2);
7676
7677 /* Construct the stack frame for the interrupt/exception handler. */
7678 VBOXSTRICTRC rcStrict;
7679 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7680 if (rcStrict == VINF_SUCCESS)
7681 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7682 if (rcStrict == VINF_SUCCESS)
7683 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7684
7685 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7686 if (rcStrict == VINF_SUCCESS)
7687 {
7688 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7689 pCtx->rip = IdtEntry.offSel;
7690 pCtx->cs.Sel = IdtEntry.uSel;
7691 pCtx->cs.ValidSel = IdtEntry.uSel;
7692 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7693 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7694 && uVector == X86_XCPT_PF)
7695 pCtx->cr2 = GCPtrFaultAddress;
7696
7697 /* If any other guest-state bits are changed here, make sure to update
7698 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7699 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7700 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7701 | HM_CHANGED_GUEST_RSP);
7702
7703 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7704 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7705 {
7706 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7707 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7708 Log4Func(("Clearing inhibition due to STI\n"));
7709 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7710 }
7711 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7712 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7713
7714 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7715 it, if we are returning to ring-3 before executing guest code. */
7716 pVCpu->hm.s.Event.fPending = false;
7717
7718 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7719 if (fStepping)
7720 rcStrict = VINF_EM_DBG_STEPPED;
7721 }
7722 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7723 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7724 return rcStrict;
7725 }
7726 }
7727
7728 /* Validate. */
7729 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7730 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7731 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7732
7733 /* Inject. */
7734 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7735 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7736 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7737 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7738 AssertRCReturn(rc, rc);
7739
7740 /* Update CR2. */
7741 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7742 && uVector == X86_XCPT_PF)
7743 pCtx->cr2 = GCPtrFaultAddress;
7744
7745 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7746
7747 return VINF_SUCCESS;
7748}
7749
7750
7751/**
7752 * Clears the interrupt-window exiting control in the VMCS and if necessary
7753 * clears the current event in the VMCS as well.
7754 *
7755 * @returns VBox status code.
7756 * @param pVCpu The cross context virtual CPU structure.
7757 *
7758 * @remarks Use this function only to clear events that have not yet been
7759 * delivered to the guest but are injected in the VMCS!
7760 * @remarks No-long-jump zone!!!
7761 */
7762static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7763{
7764 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7765 {
7766 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7767 Log4Func(("Cleared interrupt widow\n"));
7768 }
7769
7770 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7771 {
7772 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7773 Log4Func(("Cleared interrupt widow\n"));
7774 }
7775}
7776
7777
7778/**
7779 * Enters the VT-x session.
7780 *
7781 * @returns VBox status code.
7782 * @param pVCpu The cross context virtual CPU structure.
7783 * @param pHostCpu Pointer to the global CPU info struct.
7784 */
7785VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7786{
7787 AssertPtr(pVCpu);
7788 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7789 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7790 RT_NOREF(pHostCpu);
7791
7792 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7793 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7794 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7795
7796#ifdef VBOX_STRICT
7797 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7798 RTCCUINTREG uHostCR4 = ASMGetCR4();
7799 if (!(uHostCR4 & X86_CR4_VMXE))
7800 {
7801 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7802 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7803 }
7804#endif
7805
7806 /*
7807 * Load the VCPU's VMCS as the current (and active) one.
7808 */
7809 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7810 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7811 if (RT_FAILURE(rc))
7812 return rc;
7813
7814 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7815 pVCpu->hm.s.fLeaveDone = false;
7816 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7817
7818 return VINF_SUCCESS;
7819}
7820
7821
7822/**
7823 * The thread-context callback (only on platforms which support it).
7824 *
7825 * @param enmEvent The thread-context event.
7826 * @param pVCpu The cross context virtual CPU structure.
7827 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7828 * @thread EMT(pVCpu)
7829 */
7830VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7831{
7832 NOREF(fGlobalInit);
7833
7834 switch (enmEvent)
7835 {
7836 case RTTHREADCTXEVENT_OUT:
7837 {
7838 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7839 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7840 VMCPU_ASSERT_EMT(pVCpu);
7841
7842 /* No longjmps (logger flushes, locks) in this fragile context. */
7843 VMMRZCallRing3Disable(pVCpu);
7844 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7845
7846 /*
7847 * Restore host-state (FPU, debug etc.)
7848 */
7849 if (!pVCpu->hm.s.fLeaveDone)
7850 {
7851 /*
7852 * Do -not- import the guest-state here as we might already be in the middle of importing
7853 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7854 */
7855 hmR0VmxLeave(pVCpu, false /* fImportState */);
7856 pVCpu->hm.s.fLeaveDone = true;
7857 }
7858
7859 /* Leave HM context, takes care of local init (term). */
7860 int rc = HMR0LeaveCpu(pVCpu);
7861 AssertRC(rc); NOREF(rc);
7862
7863 /* Restore longjmp state. */
7864 VMMRZCallRing3Enable(pVCpu);
7865 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7866 break;
7867 }
7868
7869 case RTTHREADCTXEVENT_IN:
7870 {
7871 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7872 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7873 VMCPU_ASSERT_EMT(pVCpu);
7874
7875 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7876 VMMRZCallRing3Disable(pVCpu);
7877 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7878
7879 /* Initialize the bare minimum state required for HM. This takes care of
7880 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7881 int rc = hmR0EnterCpu(pVCpu);
7882 AssertRC(rc);
7883 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7884 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7885
7886 /* Load the active VMCS as the current one. */
7887 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7888 {
7889 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7890 AssertRC(rc); NOREF(rc);
7891 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7892 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7893 }
7894 pVCpu->hm.s.fLeaveDone = false;
7895
7896 /* Restore longjmp state. */
7897 VMMRZCallRing3Enable(pVCpu);
7898 break;
7899 }
7900
7901 default:
7902 break;
7903 }
7904}
7905
7906
7907/**
7908 * Exports the host state into the VMCS host-state area.
7909 * Sets up the VM-exit MSR-load area.
7910 *
7911 * The CPU state will be loaded from these fields on every successful VM-exit.
7912 *
7913 * @returns VBox status code.
7914 * @param pVCpu The cross context virtual CPU structure.
7915 *
7916 * @remarks No-long-jump zone!!!
7917 */
7918static int hmR0VmxExportHostState(PVMCPU pVCpu)
7919{
7920 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7921
7922 int rc = VINF_SUCCESS;
7923 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7924 {
7925 rc = hmR0VmxExportHostControlRegs();
7926 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7927
7928 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7929 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7930
7931 rc = hmR0VmxExportHostMsrs(pVCpu);
7932 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7933
7934 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7935 }
7936 return rc;
7937}
7938
7939
7940/**
7941 * Saves the host state in the VMCS host-state.
7942 *
7943 * @returns VBox status code.
7944 * @param pVCpu The cross context virtual CPU structure.
7945 *
7946 * @remarks No-long-jump zone!!!
7947 */
7948VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7949{
7950 AssertPtr(pVCpu);
7951 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7952
7953 /*
7954 * Export the host state here while entering HM context.
7955 * When thread-context hooks are used, we might get preempted and have to re-save the host
7956 * state but most of the time we won't be, so do it here before we disable interrupts.
7957 */
7958 return hmR0VmxExportHostState(pVCpu);
7959}
7960
7961
7962/**
7963 * Exports the guest state into the VMCS guest-state area.
7964 *
7965 * The will typically be done before VM-entry when the guest-CPU state and the
7966 * VMCS state may potentially be out of sync.
7967 *
7968 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7969 * VM-entry controls.
7970 * Sets up the appropriate VMX non-root function to execute guest code based on
7971 * the guest CPU mode.
7972 *
7973 * @returns VBox strict status code.
7974 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7975 * without unrestricted guest access and the VMMDev is not presently
7976 * mapped (e.g. EFI32).
7977 *
7978 * @param pVCpu The cross context virtual CPU structure.
7979 *
7980 * @remarks No-long-jump zone!!!
7981 */
7982static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7983{
7984 AssertPtr(pVCpu);
7985 HMVMX_ASSERT_PREEMPT_SAFE();
7986
7987 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7988
7989 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7990
7991 /* Determine real-on-v86 mode. */
7992 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7993 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7994 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7995 {
7996 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7997 }
7998
7999 /*
8000 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8001 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8002 */
8003 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
8004 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8005
8006 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8007 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
8008 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8009
8010 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8011 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8012 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8013
8014 rc = hmR0VmxExportGuestCR0(pVCpu);
8015 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8016
8017 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8018 if (rcStrict == VINF_SUCCESS)
8019 { /* likely */ }
8020 else
8021 {
8022 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8023 return rcStrict;
8024 }
8025
8026 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8027 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8028
8029 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8030 may alter controls if we determine we don't have to swap EFER after all. */
8031 rc = hmR0VmxExportGuestMsrs(pVCpu);
8032 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8033
8034 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8035 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8036
8037 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8038 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8039
8040 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8041 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8042 rc = hmR0VmxExportGuestRip(pVCpu);
8043 rc |= hmR0VmxExportGuestRsp(pVCpu);
8044 rc |= hmR0VmxExportGuestRflags(pVCpu);
8045 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8046
8047 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8048 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8049 | HM_CHANGED_GUEST_CR2
8050 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8051 | HM_CHANGED_GUEST_X87
8052 | HM_CHANGED_GUEST_SSE_AVX
8053 | HM_CHANGED_GUEST_OTHER_XSAVE
8054 | HM_CHANGED_GUEST_XCRx
8055 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8056 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8057 | HM_CHANGED_GUEST_TSC_AUX
8058 | HM_CHANGED_GUEST_OTHER_MSRS
8059 | HM_CHANGED_GUEST_HWVIRT
8060 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8061
8062 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8063 return rc;
8064}
8065
8066
8067/**
8068 * Exports the state shared between the host and guest into the VMCS.
8069 *
8070 * @param pVCpu The cross context virtual CPU structure.
8071 *
8072 * @remarks No-long-jump zone!!!
8073 */
8074static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8075{
8076 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8077 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8078
8079 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8080 {
8081 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8082 AssertRC(rc);
8083 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8084
8085 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8086 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8087 {
8088 rc = hmR0VmxExportGuestRflags(pVCpu);
8089 AssertRC(rc);
8090 }
8091 }
8092
8093 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8094 {
8095 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8096 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8097 }
8098
8099 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8100 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8101}
8102
8103
8104/**
8105 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8106 *
8107 * @returns Strict VBox status code (i.e. informational status codes too).
8108 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8109 * without unrestricted guest access and the VMMDev is not presently
8110 * mapped (e.g. EFI32).
8111 *
8112 * @param pVCpu The cross context virtual CPU structure.
8113 *
8114 * @remarks No-long-jump zone!!!
8115 */
8116static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8117{
8118 HMVMX_ASSERT_PREEMPT_SAFE();
8119 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8120 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8121
8122#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8123 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8124#endif
8125
8126 /*
8127 * For many exits it's only RIP that changes and hence try to export it first
8128 * without going through a lot of change flag checks.
8129 */
8130 VBOXSTRICTRC rcStrict;
8131 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8132 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8133 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8134 {
8135 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8136 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8137 { /* likely */}
8138 else
8139 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8140 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8141 }
8142 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8143 {
8144 rcStrict = hmR0VmxExportGuestState(pVCpu);
8145 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8146 { /* likely */}
8147 else
8148 {
8149 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8150 VBOXSTRICTRC_VAL(rcStrict)));
8151 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8152 return rcStrict;
8153 }
8154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8155 }
8156 else
8157 rcStrict = VINF_SUCCESS;
8158
8159#ifdef VBOX_STRICT
8160 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8161 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8162 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8163 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8164 ("fCtxChanged=%#RX64\n", fCtxChanged));
8165#endif
8166 return rcStrict;
8167}
8168
8169
8170/**
8171 * Does the preparations before executing guest code in VT-x.
8172 *
8173 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8174 * recompiler/IEM. We must be cautious what we do here regarding committing
8175 * guest-state information into the VMCS assuming we assuredly execute the
8176 * guest in VT-x mode.
8177 *
8178 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8179 * the common-state (TRPM/forceflags), we must undo those changes so that the
8180 * recompiler/IEM can (and should) use them when it resumes guest execution.
8181 * Otherwise such operations must be done when we can no longer exit to ring-3.
8182 *
8183 * @returns Strict VBox status code (i.e. informational status codes too).
8184 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8185 * have been disabled.
8186 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8187 * double-fault into the guest.
8188 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8189 * dispatched directly.
8190 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8191 *
8192 * @param pVCpu The cross context virtual CPU structure.
8193 * @param pVmxTransient Pointer to the VMX transient structure.
8194 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8195 * us ignore some of the reasons for returning to
8196 * ring-3, and return VINF_EM_DBG_STEPPED if event
8197 * dispatching took place.
8198 */
8199static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8200{
8201 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8202
8203#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8204 PGMRZDynMapFlushAutoSet(pVCpu);
8205#endif
8206
8207 /* Check force flag actions that might require us to go back to ring-3. */
8208 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8209 if (rcStrict == VINF_SUCCESS)
8210 { /* FFs doesn't get set all the time. */ }
8211 else
8212 return rcStrict;
8213
8214 /*
8215 * Setup the virtualized-APIC accesses.
8216 *
8217 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8218 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8219 *
8220 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8221 */
8222 PVM pVM = pVCpu->CTX_SUFF(pVM);
8223 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8224 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8225 && PDMHasApic(pVM))
8226 {
8227 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8228 Assert(u64MsrApicBase);
8229 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8230
8231 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8232
8233 /* Unalias any existing mapping. */
8234 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8235 AssertRCReturn(rc, rc);
8236
8237 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8238 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8239 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8240 AssertRCReturn(rc, rc);
8241
8242 /* Update the per-VCPU cache of the APIC base MSR. */
8243 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8244 }
8245
8246 if (TRPMHasTrap(pVCpu))
8247 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8248 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8249
8250 /*
8251 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8252 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8253 * also result in triple-faulting the VM.
8254 */
8255 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8256 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8257 { /* likely */ }
8258 else
8259 {
8260 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8261 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8262 return rcStrict;
8263 }
8264
8265 /*
8266 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8267 * import CR3 themselves. We will need to update them here as even as late as the above
8268 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8269 * the below force flags to be set.
8270 */
8271 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8272 {
8273 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8274 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8275 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8276 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8277 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8278 }
8279 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8280 {
8281 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8282 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8283 }
8284
8285 /*
8286 * No longjmps to ring-3 from this point on!!!
8287 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8288 * This also disables flushing of the R0-logger instance (if any).
8289 */
8290 VMMRZCallRing3Disable(pVCpu);
8291
8292 /*
8293 * Export the guest state bits.
8294 *
8295 * We cannot perform longjmps while loading the guest state because we do not preserve the
8296 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8297 * CPU migration.
8298 *
8299 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8300 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8301 * Hence, loading of the guest state needs to be done -after- injection of events.
8302 */
8303 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8304 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8305 { /* likely */ }
8306 else
8307 {
8308 VMMRZCallRing3Enable(pVCpu);
8309 return rcStrict;
8310 }
8311
8312 /*
8313 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8314 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8315 * preemption disabled for a while. Since this is purly to aid the
8316 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8317 * disable interrupt on NT.
8318 *
8319 * We need to check for force-flags that could've possible been altered since we last
8320 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8321 * see @bugref{6398}).
8322 *
8323 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8324 * to ring-3 before executing guest code.
8325 */
8326 pVmxTransient->fEFlags = ASMIntDisableFlags();
8327
8328 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8329 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8330 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8331 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8332 {
8333 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8334 {
8335 pVCpu->hm.s.Event.fPending = false;
8336
8337 /*
8338 * We've injected any pending events. This is really the point of no return (to ring-3).
8339 *
8340 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8341 * returns from this function, so don't enable them here.
8342 */
8343 return VINF_SUCCESS;
8344 }
8345
8346 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8347 rcStrict = VINF_EM_RAW_INTERRUPT;
8348 }
8349 else
8350 {
8351 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8352 rcStrict = VINF_EM_RAW_TO_R3;
8353 }
8354
8355 ASMSetFlags(pVmxTransient->fEFlags);
8356 VMMRZCallRing3Enable(pVCpu);
8357
8358 return rcStrict;
8359}
8360
8361
8362/**
8363 * Prepares to run guest code in VT-x and we've committed to doing so. This
8364 * means there is no backing out to ring-3 or anywhere else at this
8365 * point.
8366 *
8367 * @param pVCpu The cross context virtual CPU structure.
8368 * @param pVmxTransient Pointer to the VMX transient structure.
8369 *
8370 * @remarks Called with preemption disabled.
8371 * @remarks No-long-jump zone!!!
8372 */
8373static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8374{
8375 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8376 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8377 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8378
8379 /*
8380 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8381 */
8382 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8383 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8384
8385 PVM pVM = pVCpu->CTX_SUFF(pVM);
8386 if (!CPUMIsGuestFPUStateActive(pVCpu))
8387 {
8388 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8389 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8390 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8391 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8392 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8393 }
8394
8395 /*
8396 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8397 */
8398 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8399 && pVCpu->hm.s.vmx.cMsrs > 0)
8400 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8401
8402 /*
8403 * Re-save the host state bits as we may've been preempted (only happens when
8404 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8405 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8406 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8407 * See @bugref{8432}.
8408 */
8409 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8410 {
8411 int rc = hmR0VmxExportHostState(pVCpu);
8412 AssertRC(rc);
8413 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8414 }
8415 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8416
8417 /*
8418 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8419 */
8420 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8421 hmR0VmxExportSharedState(pVCpu);
8422 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8423
8424 /* Store status of the shared guest-host state at the time of VM-entry. */
8425#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8426 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8427 {
8428 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8429 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8430 }
8431 else
8432#endif
8433 {
8434 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8435 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8436 }
8437
8438 /*
8439 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8440 */
8441 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8442 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8443
8444 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8445 RTCPUID idCurrentCpu = pCpu->idCpu;
8446 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8447 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8448 {
8449 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8450 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8451 }
8452
8453 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8454 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8455 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8456 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8457
8458 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8459
8460 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8461 to start executing. */
8462
8463 /*
8464 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8465 */
8466 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8467 {
8468 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8469 {
8470 bool fMsrUpdated;
8471 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8472 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8473 &fMsrUpdated);
8474 AssertRC(rc2);
8475 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8476 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8477 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8478 }
8479 else
8480 {
8481 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8482 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8483 }
8484 }
8485
8486 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8487 {
8488 bool fMsrUpdated;
8489 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8490 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8491 &fMsrUpdated);
8492 AssertRC(rc2);
8493 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8494 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8495 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8496 }
8497
8498#ifdef VBOX_STRICT
8499 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8500 hmR0VmxCheckHostEferMsr(pVCpu);
8501 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8502#endif
8503#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8504 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8505 {
8506 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8507 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8508 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8509 }
8510#endif
8511}
8512
8513
8514/**
8515 * Performs some essential restoration of state after running guest code in
8516 * VT-x.
8517 *
8518 * @param pVCpu The cross context virtual CPU structure.
8519 * @param pVmxTransient Pointer to the VMX transient structure.
8520 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8521 *
8522 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8523 *
8524 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8525 * unconditionally when it is safe to do so.
8526 */
8527static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8528{
8529 uint64_t const uHostTsc = ASMReadTSC();
8530 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8531
8532 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8533 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8534 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8535 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8536 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8537 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8538
8539 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8540 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8541
8542 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8543 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8544 Assert(!ASMIntAreEnabled());
8545 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8546
8547#if HC_ARCH_BITS == 64
8548 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8549#endif
8550#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8551 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8552 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8553 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8554#else
8555 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8556#endif
8557#ifdef VBOX_STRICT
8558 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8559#endif
8560 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8561
8562 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8563 uint32_t uExitReason;
8564 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8565 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8566 AssertRC(rc);
8567 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8568 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8569
8570 if (rcVMRun == VINF_SUCCESS)
8571 {
8572 /*
8573 * Update the VM-exit history array here even if the VM-entry failed due to:
8574 * - Invalid guest state.
8575 * - MSR loading.
8576 * - Machine-check event.
8577 *
8578 * In any of the above cases we will still have a "valid" VM-exit reason
8579 * despite @a fVMEntryFailed being false.
8580 *
8581 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8582 *
8583 * Note! We don't have CS or RIP at this point. Will probably address that later
8584 * by amending the history entry added here.
8585 */
8586 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8587 UINT64_MAX, uHostTsc);
8588
8589 if (!pVmxTransient->fVMEntryFailed)
8590 {
8591 VMMRZCallRing3Enable(pVCpu);
8592
8593 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8594 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8595
8596#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8597 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8598 AssertRC(rc);
8599#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8600 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8601 AssertRC(rc);
8602#else
8603 /*
8604 * Import the guest-interruptibility state always as we need it while evaluating
8605 * injecting events on re-entry.
8606 *
8607 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8608 * checking for real-mode while exporting the state because all bits that cause
8609 * mode changes wrt CR0 are intercepted.
8610 */
8611 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8612 AssertRC(rc);
8613#endif
8614
8615 /*
8616 * Sync the TPR shadow with our APIC state.
8617 */
8618 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8619 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8620 {
8621 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8622 AssertRC(rc);
8623 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8624 }
8625
8626 return;
8627 }
8628 }
8629 else
8630 {
8631 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8632 }
8633
8634 VMMRZCallRing3Enable(pVCpu);
8635}
8636
8637
8638/**
8639 * Runs the guest code using VT-x the normal way.
8640 *
8641 * @returns VBox status code.
8642 * @param pVCpu The cross context virtual CPU structure.
8643 *
8644 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8645 */
8646static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8647{
8648 VMXTRANSIENT VmxTransient;
8649 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8650 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8651 uint32_t cLoops = 0;
8652
8653 for (;; cLoops++)
8654 {
8655 Assert(!HMR0SuspendPending());
8656 HMVMX_ASSERT_CPU_SAFE();
8657
8658 /* Preparatory work for running guest code, this may force us to return
8659 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8660 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8661 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8662 if (rcStrict != VINF_SUCCESS)
8663 break;
8664
8665 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8666 int rcRun = hmR0VmxRunGuest(pVCpu);
8667
8668 /* Restore any residual host-state and save any bits shared between host
8669 and guest into the guest-CPU state. Re-enables interrupts! */
8670 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8671
8672 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8673 if (RT_SUCCESS(rcRun))
8674 { /* very likely */ }
8675 else
8676 {
8677 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8678 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8679 return rcRun;
8680 }
8681
8682 /* Profile the VM-exit. */
8683 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8684 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8685 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8686 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8687 HMVMX_START_EXIT_DISPATCH_PROF();
8688
8689 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8690
8691 /* Handle the VM-exit. */
8692#ifdef HMVMX_USE_FUNCTION_TABLE
8693 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8694#else
8695 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8696#endif
8697 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8698 if (rcStrict == VINF_SUCCESS)
8699 {
8700 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8701 continue; /* likely */
8702 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8703 rcStrict = VINF_EM_RAW_INTERRUPT;
8704 }
8705 break;
8706 }
8707
8708 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8709 return rcStrict;
8710}
8711
8712
8713
8714/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8715 * probes.
8716 *
8717 * The following few functions and associated structure contains the bloat
8718 * necessary for providing detailed debug events and dtrace probes as well as
8719 * reliable host side single stepping. This works on the principle of
8720 * "subclassing" the normal execution loop and workers. We replace the loop
8721 * method completely and override selected helpers to add necessary adjustments
8722 * to their core operation.
8723 *
8724 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8725 * any performance for debug and analysis features.
8726 *
8727 * @{
8728 */
8729
8730/**
8731 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8732 * the debug run loop.
8733 */
8734typedef struct VMXRUNDBGSTATE
8735{
8736 /** The RIP we started executing at. This is for detecting that we stepped. */
8737 uint64_t uRipStart;
8738 /** The CS we started executing with. */
8739 uint16_t uCsStart;
8740
8741 /** Whether we've actually modified the 1st execution control field. */
8742 bool fModifiedProcCtls : 1;
8743 /** Whether we've actually modified the 2nd execution control field. */
8744 bool fModifiedProcCtls2 : 1;
8745 /** Whether we've actually modified the exception bitmap. */
8746 bool fModifiedXcptBitmap : 1;
8747
8748 /** We desire the modified the CR0 mask to be cleared. */
8749 bool fClearCr0Mask : 1;
8750 /** We desire the modified the CR4 mask to be cleared. */
8751 bool fClearCr4Mask : 1;
8752 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8753 uint32_t fCpe1Extra;
8754 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8755 uint32_t fCpe1Unwanted;
8756 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8757 uint32_t fCpe2Extra;
8758 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8759 uint32_t bmXcptExtra;
8760 /** The sequence number of the Dtrace provider settings the state was
8761 * configured against. */
8762 uint32_t uDtraceSettingsSeqNo;
8763 /** VM-exits to check (one bit per VM-exit). */
8764 uint32_t bmExitsToCheck[3];
8765
8766 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8767 uint32_t fProcCtlsInitial;
8768 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8769 uint32_t fProcCtls2Initial;
8770 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8771 uint32_t bmXcptInitial;
8772} VMXRUNDBGSTATE;
8773AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8774typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8775
8776
8777/**
8778 * Initializes the VMXRUNDBGSTATE structure.
8779 *
8780 * @param pVCpu The cross context virtual CPU structure of the
8781 * calling EMT.
8782 * @param pDbgState The structure to initialize.
8783 */
8784static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8785{
8786 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8787 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8788
8789 pDbgState->fModifiedProcCtls = false;
8790 pDbgState->fModifiedProcCtls2 = false;
8791 pDbgState->fModifiedXcptBitmap = false;
8792 pDbgState->fClearCr0Mask = false;
8793 pDbgState->fClearCr4Mask = false;
8794 pDbgState->fCpe1Extra = 0;
8795 pDbgState->fCpe1Unwanted = 0;
8796 pDbgState->fCpe2Extra = 0;
8797 pDbgState->bmXcptExtra = 0;
8798 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8799 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8800 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8801}
8802
8803
8804/**
8805 * Updates the VMSC fields with changes requested by @a pDbgState.
8806 *
8807 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8808 * immediately before executing guest code, i.e. when interrupts are disabled.
8809 * We don't check status codes here as we cannot easily assert or return in the
8810 * latter case.
8811 *
8812 * @param pVCpu The cross context virtual CPU structure.
8813 * @param pDbgState The debug state.
8814 */
8815static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8816{
8817 /*
8818 * Ensure desired flags in VMCS control fields are set.
8819 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8820 *
8821 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8822 * there should be no stale data in pCtx at this point.
8823 */
8824 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8825 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8826 {
8827 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8828 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8829 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8830 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8831 pDbgState->fModifiedProcCtls = true;
8832 }
8833
8834 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8835 {
8836 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8837 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8838 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8839 pDbgState->fModifiedProcCtls2 = true;
8840 }
8841
8842 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8843 {
8844 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8845 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8846 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8847 pDbgState->fModifiedXcptBitmap = true;
8848 }
8849
8850 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8851 {
8852 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8853 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8854 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8855 }
8856
8857 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8858 {
8859 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8860 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8861 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8862 }
8863}
8864
8865
8866static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8867{
8868 /*
8869 * Restore VM-exit control settings as we may not reenter this function the
8870 * next time around.
8871 */
8872 /* We reload the initial value, trigger what we can of recalculations the
8873 next time around. From the looks of things, that's all that's required atm. */
8874 if (pDbgState->fModifiedProcCtls)
8875 {
8876 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8877 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8878 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8879 AssertRCReturn(rc2, rc2);
8880 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8881 }
8882
8883 /* We're currently the only ones messing with this one, so just restore the
8884 cached value and reload the field. */
8885 if ( pDbgState->fModifiedProcCtls2
8886 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8887 {
8888 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8889 AssertRCReturn(rc2, rc2);
8890 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8891 }
8892
8893 /* If we've modified the exception bitmap, we restore it and trigger
8894 reloading and partial recalculation the next time around. */
8895 if (pDbgState->fModifiedXcptBitmap)
8896 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8897
8898 return rcStrict;
8899}
8900
8901
8902/**
8903 * Configures VM-exit controls for current DBGF and DTrace settings.
8904 *
8905 * This updates @a pDbgState and the VMCS execution control fields to reflect
8906 * the necessary VM-exits demanded by DBGF and DTrace.
8907 *
8908 * @param pVCpu The cross context virtual CPU structure.
8909 * @param pDbgState The debug state.
8910 * @param pVmxTransient Pointer to the VMX transient structure. May update
8911 * fUpdateTscOffsettingAndPreemptTimer.
8912 */
8913static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8914{
8915 /*
8916 * Take down the dtrace serial number so we can spot changes.
8917 */
8918 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8919 ASMCompilerBarrier();
8920
8921 /*
8922 * We'll rebuild most of the middle block of data members (holding the
8923 * current settings) as we go along here, so start by clearing it all.
8924 */
8925 pDbgState->bmXcptExtra = 0;
8926 pDbgState->fCpe1Extra = 0;
8927 pDbgState->fCpe1Unwanted = 0;
8928 pDbgState->fCpe2Extra = 0;
8929 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8930 pDbgState->bmExitsToCheck[i] = 0;
8931
8932 /*
8933 * Software interrupts (INT XXh) - no idea how to trigger these...
8934 */
8935 PVM pVM = pVCpu->CTX_SUFF(pVM);
8936 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8937 || VBOXVMM_INT_SOFTWARE_ENABLED())
8938 {
8939 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8940 }
8941
8942 /*
8943 * INT3 breakpoints - triggered by #BP exceptions.
8944 */
8945 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8946 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8947
8948 /*
8949 * Exception bitmap and XCPT events+probes.
8950 */
8951 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8952 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8953 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8954
8955 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8956 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8957 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8958 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8959 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8960 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8961 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8962 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8963 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8964 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8965 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8966 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8967 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8968 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8969 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8970 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8971 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8972 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8973
8974 if (pDbgState->bmXcptExtra)
8975 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8976
8977 /*
8978 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8979 *
8980 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8981 * So, when adding/changing/removing please don't forget to update it.
8982 *
8983 * Some of the macros are picking up local variables to save horizontal space,
8984 * (being able to see it in a table is the lesser evil here).
8985 */
8986#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8987 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8988 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8989#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8990 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8991 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8992 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8993 } else do { } while (0)
8994#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8995 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8996 { \
8997 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8998 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8999 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9000 } else do { } while (0)
9001#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9002 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9003 { \
9004 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9005 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9006 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9007 } else do { } while (0)
9008#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9009 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9010 { \
9011 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9012 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9013 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9014 } else do { } while (0)
9015
9016 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9017 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9018 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9019 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9020 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9021
9022 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9023 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9024 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9025 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9026 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9027 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9028 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9029 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9030 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9031 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9032 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9033 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9034 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9035 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9036 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9037 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9038 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9039 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9040 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9041 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9042 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9043 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9044 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9045 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9046 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9047 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9048 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9049 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9050 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9051 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9052 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9053 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9054 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9055 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9056 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9057 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9058
9059 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9060 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9061 {
9062 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9063 AssertRC(rc);
9064
9065#if 0 /** @todo fix me */
9066 pDbgState->fClearCr0Mask = true;
9067 pDbgState->fClearCr4Mask = true;
9068#endif
9069 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9070 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9071 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9072 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9073 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9074 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9075 require clearing here and in the loop if we start using it. */
9076 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9077 }
9078 else
9079 {
9080 if (pDbgState->fClearCr0Mask)
9081 {
9082 pDbgState->fClearCr0Mask = false;
9083 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9084 }
9085 if (pDbgState->fClearCr4Mask)
9086 {
9087 pDbgState->fClearCr4Mask = false;
9088 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9089 }
9090 }
9091 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9092 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9093
9094 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9095 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9096 {
9097 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9098 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9099 }
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9101 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9102
9103 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9104 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9105 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9106 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9107 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9108 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9109 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9110 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9111#if 0 /** @todo too slow, fix handler. */
9112 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9113#endif
9114 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9115
9116 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9117 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9118 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9119 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9120 {
9121 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9122 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9123 }
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9125 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9127 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9128
9129 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9130 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9131 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9132 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9133 {
9134 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9135 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9136 }
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9139 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9141
9142 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9144 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9146 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9148 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9150 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9152 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9154 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9156 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9158 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9160 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9161 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9162 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9164
9165#undef IS_EITHER_ENABLED
9166#undef SET_ONLY_XBM_IF_EITHER_EN
9167#undef SET_CPE1_XBM_IF_EITHER_EN
9168#undef SET_CPEU_XBM_IF_EITHER_EN
9169#undef SET_CPE2_XBM_IF_EITHER_EN
9170
9171 /*
9172 * Sanitize the control stuff.
9173 */
9174 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9175 if (pDbgState->fCpe2Extra)
9176 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9177 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9178 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9179 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9180 {
9181 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9182 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9183 }
9184
9185 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9186 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9187 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9188 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9189}
9190
9191
9192/**
9193 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9194 * appropriate.
9195 *
9196 * The caller has checked the VM-exit against the
9197 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9198 * already, so we don't have to do that either.
9199 *
9200 * @returns Strict VBox status code (i.e. informational status codes too).
9201 * @param pVCpu The cross context virtual CPU structure.
9202 * @param pVmxTransient Pointer to the VMX-transient structure.
9203 * @param uExitReason The VM-exit reason.
9204 *
9205 * @remarks The name of this function is displayed by dtrace, so keep it short
9206 * and to the point. No longer than 33 chars long, please.
9207 */
9208static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9209{
9210 /*
9211 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9212 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9213 *
9214 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9215 * does. Must add/change/remove both places. Same ordering, please.
9216 *
9217 * Added/removed events must also be reflected in the next section
9218 * where we dispatch dtrace events.
9219 */
9220 bool fDtrace1 = false;
9221 bool fDtrace2 = false;
9222 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9223 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9224 uint32_t uEventArg = 0;
9225#define SET_EXIT(a_EventSubName) \
9226 do { \
9227 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9228 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9229 } while (0)
9230#define SET_BOTH(a_EventSubName) \
9231 do { \
9232 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9233 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9234 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9235 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9236 } while (0)
9237 switch (uExitReason)
9238 {
9239 case VMX_EXIT_MTF:
9240 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9241
9242 case VMX_EXIT_XCPT_OR_NMI:
9243 {
9244 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9245 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9246 {
9247 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9248 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9249 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9250 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9251 {
9252 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9253 {
9254 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9255 uEventArg = pVmxTransient->uExitIntErrorCode;
9256 }
9257 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9258 switch (enmEvent1)
9259 {
9260 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9261 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9262 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9263 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9264 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9265 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9266 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9267 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9268 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9269 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9270 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9271 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9272 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9273 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9274 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9275 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9276 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9277 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9278 default: break;
9279 }
9280 }
9281 else
9282 AssertFailed();
9283 break;
9284
9285 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9286 uEventArg = idxVector;
9287 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9288 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9289 break;
9290 }
9291 break;
9292 }
9293
9294 case VMX_EXIT_TRIPLE_FAULT:
9295 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9296 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9297 break;
9298 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9299 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9300 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9301 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9302 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9303
9304 /* Instruction specific VM-exits: */
9305 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9306 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9307 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9308 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9309 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9310 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9311 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9312 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9313 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9314 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9315 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9316 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9317 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9318 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9319 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9320 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9321 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9322 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9323 case VMX_EXIT_MOV_CRX:
9324 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9325 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9326 SET_BOTH(CRX_READ);
9327 else
9328 SET_BOTH(CRX_WRITE);
9329 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9330 break;
9331 case VMX_EXIT_MOV_DRX:
9332 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9333 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9334 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9335 SET_BOTH(DRX_READ);
9336 else
9337 SET_BOTH(DRX_WRITE);
9338 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9339 break;
9340 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9341 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9342 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9343 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9344 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9345 case VMX_EXIT_XDTR_ACCESS:
9346 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9347 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9348 {
9349 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9350 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9351 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9352 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9353 }
9354 break;
9355
9356 case VMX_EXIT_TR_ACCESS:
9357 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9358 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9359 {
9360 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9361 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9362 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9363 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9364 }
9365 break;
9366
9367 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9368 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9369 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9370 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9371 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9372 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9373 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9374 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9375 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9376 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9377 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9378
9379 /* Events that aren't relevant at this point. */
9380 case VMX_EXIT_EXT_INT:
9381 case VMX_EXIT_INT_WINDOW:
9382 case VMX_EXIT_NMI_WINDOW:
9383 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9384 case VMX_EXIT_PREEMPT_TIMER:
9385 case VMX_EXIT_IO_INSTR:
9386 break;
9387
9388 /* Errors and unexpected events. */
9389 case VMX_EXIT_INIT_SIGNAL:
9390 case VMX_EXIT_SIPI:
9391 case VMX_EXIT_IO_SMI:
9392 case VMX_EXIT_SMI:
9393 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9394 case VMX_EXIT_ERR_MSR_LOAD:
9395 case VMX_EXIT_ERR_MACHINE_CHECK:
9396 break;
9397
9398 default:
9399 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9400 break;
9401 }
9402#undef SET_BOTH
9403#undef SET_EXIT
9404
9405 /*
9406 * Dtrace tracepoints go first. We do them here at once so we don't
9407 * have to copy the guest state saving and stuff a few dozen times.
9408 * Down side is that we've got to repeat the switch, though this time
9409 * we use enmEvent since the probes are a subset of what DBGF does.
9410 */
9411 if (fDtrace1 || fDtrace2)
9412 {
9413 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9414 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9415 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9416 switch (enmEvent1)
9417 {
9418 /** @todo consider which extra parameters would be helpful for each probe. */
9419 case DBGFEVENT_END: break;
9420 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9421 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9422 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9424 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9426 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9427 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9428 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9429 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9430 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9431 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9432 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9433 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9434 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9435 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9436 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9437 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9438 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9439 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9440 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9445 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9446 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9447 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9448 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9449 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9450 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9451 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9452 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9453 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9454 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9483 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9484 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9485 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9486 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9487 }
9488 switch (enmEvent2)
9489 {
9490 /** @todo consider which extra parameters would be helpful for each probe. */
9491 case DBGFEVENT_END: break;
9492 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9493 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9494 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9499 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9500 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9501 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9503 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9504 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9505 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9506 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9507 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9508 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9541 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9542 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9543 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9544 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9545 }
9546 }
9547
9548 /*
9549 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9550 * the DBGF call will do a full check).
9551 *
9552 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9553 * Note! If we have to events, we prioritize the first, i.e. the instruction
9554 * one, in order to avoid event nesting.
9555 */
9556 PVM pVM = pVCpu->CTX_SUFF(pVM);
9557 if ( enmEvent1 != DBGFEVENT_END
9558 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9559 {
9560 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9561 if (rcStrict != VINF_SUCCESS)
9562 return rcStrict;
9563 }
9564 else if ( enmEvent2 != DBGFEVENT_END
9565 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9566 {
9567 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9568 if (rcStrict != VINF_SUCCESS)
9569 return rcStrict;
9570 }
9571
9572 return VINF_SUCCESS;
9573}
9574
9575
9576/**
9577 * Single-stepping VM-exit filtering.
9578 *
9579 * This is preprocessing the VM-exits and deciding whether we've gotten far
9580 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9581 * handling is performed.
9582 *
9583 * @returns Strict VBox status code (i.e. informational status codes too).
9584 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9585 * @param pVmxTransient Pointer to the VMX-transient structure.
9586 * @param pDbgState The debug state.
9587 */
9588DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9589{
9590 /*
9591 * Expensive (saves context) generic dtrace VM-exit probe.
9592 */
9593 uint32_t const uExitReason = pVmxTransient->uExitReason;
9594 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9595 { /* more likely */ }
9596 else
9597 {
9598 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9599 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9600 AssertRC(rc);
9601 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9602 }
9603
9604 /*
9605 * Check for host NMI, just to get that out of the way.
9606 */
9607 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9608 { /* normally likely */ }
9609 else
9610 {
9611 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9612 AssertRCReturn(rc2, rc2);
9613 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9614 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9615 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9616 }
9617
9618 /*
9619 * Check for single stepping event if we're stepping.
9620 */
9621 if (pVCpu->hm.s.fSingleInstruction)
9622 {
9623 switch (uExitReason)
9624 {
9625 case VMX_EXIT_MTF:
9626 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9627
9628 /* Various events: */
9629 case VMX_EXIT_XCPT_OR_NMI:
9630 case VMX_EXIT_EXT_INT:
9631 case VMX_EXIT_TRIPLE_FAULT:
9632 case VMX_EXIT_INT_WINDOW:
9633 case VMX_EXIT_NMI_WINDOW:
9634 case VMX_EXIT_TASK_SWITCH:
9635 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9636 case VMX_EXIT_APIC_ACCESS:
9637 case VMX_EXIT_EPT_VIOLATION:
9638 case VMX_EXIT_EPT_MISCONFIG:
9639 case VMX_EXIT_PREEMPT_TIMER:
9640
9641 /* Instruction specific VM-exits: */
9642 case VMX_EXIT_CPUID:
9643 case VMX_EXIT_GETSEC:
9644 case VMX_EXIT_HLT:
9645 case VMX_EXIT_INVD:
9646 case VMX_EXIT_INVLPG:
9647 case VMX_EXIT_RDPMC:
9648 case VMX_EXIT_RDTSC:
9649 case VMX_EXIT_RSM:
9650 case VMX_EXIT_VMCALL:
9651 case VMX_EXIT_VMCLEAR:
9652 case VMX_EXIT_VMLAUNCH:
9653 case VMX_EXIT_VMPTRLD:
9654 case VMX_EXIT_VMPTRST:
9655 case VMX_EXIT_VMREAD:
9656 case VMX_EXIT_VMRESUME:
9657 case VMX_EXIT_VMWRITE:
9658 case VMX_EXIT_VMXOFF:
9659 case VMX_EXIT_VMXON:
9660 case VMX_EXIT_MOV_CRX:
9661 case VMX_EXIT_MOV_DRX:
9662 case VMX_EXIT_IO_INSTR:
9663 case VMX_EXIT_RDMSR:
9664 case VMX_EXIT_WRMSR:
9665 case VMX_EXIT_MWAIT:
9666 case VMX_EXIT_MONITOR:
9667 case VMX_EXIT_PAUSE:
9668 case VMX_EXIT_XDTR_ACCESS:
9669 case VMX_EXIT_TR_ACCESS:
9670 case VMX_EXIT_INVEPT:
9671 case VMX_EXIT_RDTSCP:
9672 case VMX_EXIT_INVVPID:
9673 case VMX_EXIT_WBINVD:
9674 case VMX_EXIT_XSETBV:
9675 case VMX_EXIT_RDRAND:
9676 case VMX_EXIT_INVPCID:
9677 case VMX_EXIT_VMFUNC:
9678 case VMX_EXIT_RDSEED:
9679 case VMX_EXIT_XSAVES:
9680 case VMX_EXIT_XRSTORS:
9681 {
9682 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9683 AssertRCReturn(rc, rc);
9684 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9685 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9686 return VINF_EM_DBG_STEPPED;
9687 break;
9688 }
9689
9690 /* Errors and unexpected events: */
9691 case VMX_EXIT_INIT_SIGNAL:
9692 case VMX_EXIT_SIPI:
9693 case VMX_EXIT_IO_SMI:
9694 case VMX_EXIT_SMI:
9695 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9696 case VMX_EXIT_ERR_MSR_LOAD:
9697 case VMX_EXIT_ERR_MACHINE_CHECK:
9698 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9699 break;
9700
9701 default:
9702 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9703 break;
9704 }
9705 }
9706
9707 /*
9708 * Check for debugger event breakpoints and dtrace probes.
9709 */
9710 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9711 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9712 {
9713 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9714 if (rcStrict != VINF_SUCCESS)
9715 return rcStrict;
9716 }
9717
9718 /*
9719 * Normal processing.
9720 */
9721#ifdef HMVMX_USE_FUNCTION_TABLE
9722 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9723#else
9724 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9725#endif
9726}
9727
9728
9729/**
9730 * Single steps guest code using VT-x.
9731 *
9732 * @returns Strict VBox status code (i.e. informational status codes too).
9733 * @param pVCpu The cross context virtual CPU structure.
9734 *
9735 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9736 */
9737static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9738{
9739 VMXTRANSIENT VmxTransient;
9740 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9741
9742 /* Set HMCPU indicators. */
9743 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9744 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9745 pVCpu->hm.s.fDebugWantRdTscExit = false;
9746 pVCpu->hm.s.fUsingDebugLoop = true;
9747
9748 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9749 VMXRUNDBGSTATE DbgState;
9750 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9751 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9752
9753 /*
9754 * The loop.
9755 */
9756 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9757 for (uint32_t cLoops = 0; ; cLoops++)
9758 {
9759 Assert(!HMR0SuspendPending());
9760 HMVMX_ASSERT_CPU_SAFE();
9761 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9762
9763 /*
9764 * Preparatory work for running guest code, this may force us to return
9765 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9766 */
9767 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9768 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9769 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9770 if (rcStrict != VINF_SUCCESS)
9771 break;
9772
9773 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9774 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9775
9776 /*
9777 * Now we can run the guest code.
9778 */
9779 int rcRun = hmR0VmxRunGuest(pVCpu);
9780
9781 /*
9782 * Restore any residual host-state and save any bits shared between host
9783 * and guest into the guest-CPU state. Re-enables interrupts!
9784 */
9785 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9786
9787 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9788 if (RT_SUCCESS(rcRun))
9789 { /* very likely */ }
9790 else
9791 {
9792 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9793 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9794 return rcRun;
9795 }
9796
9797 /* Profile the VM-exit. */
9798 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9800 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9801 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9802 HMVMX_START_EXIT_DISPATCH_PROF();
9803
9804 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9805
9806 /*
9807 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9808 */
9809 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9810 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9811 if (rcStrict != VINF_SUCCESS)
9812 break;
9813 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9814 {
9815 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9816 rcStrict = VINF_EM_RAW_INTERRUPT;
9817 break;
9818 }
9819
9820 /*
9821 * Stepping: Did the RIP change, if so, consider it a single step.
9822 * Otherwise, make sure one of the TFs gets set.
9823 */
9824 if (fStepping)
9825 {
9826 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9827 AssertRC(rc);
9828 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9829 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9830 {
9831 rcStrict = VINF_EM_DBG_STEPPED;
9832 break;
9833 }
9834 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9835 }
9836
9837 /*
9838 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9839 */
9840 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9841 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9842 }
9843
9844 /*
9845 * Clear the X86_EFL_TF if necessary.
9846 */
9847 if (pVCpu->hm.s.fClearTrapFlag)
9848 {
9849 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9850 AssertRC(rc);
9851 pVCpu->hm.s.fClearTrapFlag = false;
9852 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9853 }
9854 /** @todo there seems to be issues with the resume flag when the monitor trap
9855 * flag is pending without being used. Seen early in bios init when
9856 * accessing APIC page in protected mode. */
9857
9858 /*
9859 * Restore VM-exit control settings as we may not reenter this function the
9860 * next time around.
9861 */
9862 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9863
9864 /* Restore HMCPU indicators. */
9865 pVCpu->hm.s.fUsingDebugLoop = false;
9866 pVCpu->hm.s.fDebugWantRdTscExit = false;
9867 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9868
9869 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9870 return rcStrict;
9871}
9872
9873
9874/** @} */
9875
9876
9877/**
9878 * Checks if any expensive dtrace probes are enabled and we should go to the
9879 * debug loop.
9880 *
9881 * @returns true if we should use debug loop, false if not.
9882 */
9883static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9884{
9885 /* It's probably faster to OR the raw 32-bit counter variables together.
9886 Since the variables are in an array and the probes are next to one
9887 another (more or less), we have good locality. So, better read
9888 eight-nine cache lines ever time and only have one conditional, than
9889 128+ conditionals, right? */
9890 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9891 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9892 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9893 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9894 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9895 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9896 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9897 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9898 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9899 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9900 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9901 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9902 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9903 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9904 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9905 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9906 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9907 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9908 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9909 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9910 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9911 ) != 0
9912 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9913 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9914 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9915 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9916 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9917 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9918 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9919 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9920 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9921 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9922 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9923 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9924 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9925 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9926 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9927 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9928 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9929 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9930 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9932 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9934 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9935 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9936 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9937 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9938 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9939 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9940 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9941 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9942 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9943 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9955 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9956 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9957 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9958 ) != 0
9959 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9960 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9961 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9962 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9963 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9964 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9965 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9966 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9967 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9968 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9969 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9970 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9971 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9972 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9973 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9974 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9975 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9976 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9977 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9978 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9980 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9982 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9983 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9984 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9985 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9986 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9987 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9988 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9989 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9990 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9991 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10007 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10008 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10009 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10010 ) != 0;
10011}
10012
10013
10014/**
10015 * Runs the guest code using VT-x.
10016 *
10017 * @returns Strict VBox status code (i.e. informational status codes too).
10018 * @param pVCpu The cross context virtual CPU structure.
10019 */
10020VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10021{
10022 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10023 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10024 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10025 HMVMX_ASSERT_PREEMPT_SAFE();
10026
10027 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10028
10029 VBOXSTRICTRC rcStrict;
10030 if ( !pVCpu->hm.s.fUseDebugLoop
10031 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10032 && !DBGFIsStepping(pVCpu)
10033 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10034 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10035 else
10036 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10037
10038 if (rcStrict == VERR_EM_INTERPRETER)
10039 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10040 else if (rcStrict == VINF_EM_RESET)
10041 rcStrict = VINF_EM_TRIPLE_FAULT;
10042
10043 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10044 if (RT_FAILURE(rc2))
10045 {
10046 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10047 rcStrict = rc2;
10048 }
10049 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10050 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10051 return rcStrict;
10052}
10053
10054
10055#ifndef HMVMX_USE_FUNCTION_TABLE
10056DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10057{
10058#ifdef DEBUG_ramshankar
10059#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10060 do { \
10061 if (a_fSave != 0) \
10062 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10063 VBOXSTRICTRC rcStrict = a_CallExpr; \
10064 if (a_fSave != 0) \
10065 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10066 return rcStrict; \
10067 } while (0)
10068#else
10069# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10070#endif
10071 switch (rcReason)
10072 {
10073 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10074 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10075 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10076 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10077 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10078 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10079 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10080 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10081 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10082 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10083 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10084 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10085 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10086 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10087 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10088 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10089 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10090 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10091 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10092 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10093 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10094 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10095 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10096 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10097 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10098 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10099 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10100 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10101 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10102 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10103 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10104 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10105 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10106 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10107
10108 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10109 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10110 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10111 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10112 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10113 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10114 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10115 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10116 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10117
10118 case VMX_EXIT_VMCLEAR:
10119 case VMX_EXIT_VMLAUNCH:
10120 case VMX_EXIT_VMPTRLD:
10121 case VMX_EXIT_VMPTRST:
10122 case VMX_EXIT_VMREAD:
10123 case VMX_EXIT_VMRESUME:
10124 case VMX_EXIT_VMWRITE:
10125 case VMX_EXIT_VMXOFF:
10126 case VMX_EXIT_VMXON:
10127 case VMX_EXIT_INVEPT:
10128 case VMX_EXIT_INVVPID:
10129 case VMX_EXIT_VMFUNC:
10130 case VMX_EXIT_XSAVES:
10131 case VMX_EXIT_XRSTORS:
10132 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10133
10134 case VMX_EXIT_ENCLS:
10135 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10136 case VMX_EXIT_PML_FULL:
10137 default:
10138 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10139 }
10140#undef VMEXIT_CALL_RET
10141}
10142#endif /* !HMVMX_USE_FUNCTION_TABLE */
10143
10144
10145#ifdef VBOX_STRICT
10146/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10147# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10148 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10149
10150# define HMVMX_ASSERT_PREEMPT_CPUID() \
10151 do { \
10152 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10153 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10154 } while (0)
10155
10156# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10157 do { \
10158 AssertPtr(pVCpu); \
10159 AssertPtr(pVmxTransient); \
10160 Assert(pVmxTransient->fVMEntryFailed == false); \
10161 Assert(ASMIntAreEnabled()); \
10162 HMVMX_ASSERT_PREEMPT_SAFE(); \
10163 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10164 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10165 HMVMX_ASSERT_PREEMPT_SAFE(); \
10166 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10167 HMVMX_ASSERT_PREEMPT_CPUID(); \
10168 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10169 } while (0)
10170
10171# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10172 do { \
10173 Log4Func(("\n")); \
10174 } while (0)
10175#else /* nonstrict builds: */
10176# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10177 do { \
10178 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10179 RT_NOREF2(pVCpu, pVmxTransient); \
10180 } while (0)
10181# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10182#endif
10183
10184
10185/**
10186 * Advances the guest RIP by the specified number of bytes.
10187 *
10188 * @param pVCpu The cross context virtual CPU structure.
10189 * @param cbInstr Number of bytes to advance the RIP by.
10190 *
10191 * @remarks No-long-jump zone!!!
10192 */
10193DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10194{
10195 /* Advance the RIP. */
10196 pVCpu->cpum.GstCtx.rip += cbInstr;
10197 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10198
10199 /* Update interrupt inhibition. */
10200 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10201 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10202 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10203}
10204
10205
10206/**
10207 * Advances the guest RIP after reading it from the VMCS.
10208 *
10209 * @returns VBox status code, no informational status codes.
10210 * @param pVCpu The cross context virtual CPU structure.
10211 * @param pVmxTransient Pointer to the VMX transient structure.
10212 *
10213 * @remarks No-long-jump zone!!!
10214 */
10215static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10216{
10217 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10218 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10219 AssertRCReturn(rc, rc);
10220
10221 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10222
10223 /*
10224 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10225 * pending debug exception field as it takes care of priority of events.
10226 *
10227 * See Intel spec. 32.2.1 "Debug Exceptions".
10228 */
10229 if ( !pVCpu->hm.s.fSingleInstruction
10230 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10231 {
10232 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10233 AssertRCReturn(rc, rc);
10234 }
10235
10236 return VINF_SUCCESS;
10237}
10238
10239
10240/**
10241 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10242 * and update error record fields accordingly.
10243 *
10244 * @return VMX_IGS_* return codes.
10245 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10246 * wrong with the guest state.
10247 *
10248 * @param pVCpu The cross context virtual CPU structure.
10249 *
10250 * @remarks This function assumes our cache of the VMCS controls
10251 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10252 */
10253static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10254{
10255#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10256#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10257 uError = (err); \
10258 break; \
10259 } else do { } while (0)
10260
10261 int rc;
10262 PVM pVM = pVCpu->CTX_SUFF(pVM);
10263 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10264 uint32_t uError = VMX_IGS_ERROR;
10265 uint32_t u32Val;
10266 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10267
10268 do
10269 {
10270 /*
10271 * CR0.
10272 */
10273 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10274 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10275 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10276 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10277 if (fUnrestrictedGuest)
10278 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10279
10280 uint32_t u32GuestCr0;
10281 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10282 AssertRCBreak(rc);
10283 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10284 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10285 if ( !fUnrestrictedGuest
10286 && (u32GuestCr0 & X86_CR0_PG)
10287 && !(u32GuestCr0 & X86_CR0_PE))
10288 {
10289 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10290 }
10291
10292 /*
10293 * CR4.
10294 */
10295 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10296 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10297
10298 uint32_t u32GuestCr4;
10299 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10300 AssertRCBreak(rc);
10301 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10302 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10303
10304 /*
10305 * IA32_DEBUGCTL MSR.
10306 */
10307 uint64_t u64Val;
10308 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10309 AssertRCBreak(rc);
10310 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10311 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10312 {
10313 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10314 }
10315 uint64_t u64DebugCtlMsr = u64Val;
10316
10317#ifdef VBOX_STRICT
10318 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10319 AssertRCBreak(rc);
10320 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10321#endif
10322 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10323
10324 /*
10325 * RIP and RFLAGS.
10326 */
10327 uint32_t u32Eflags;
10328#if HC_ARCH_BITS == 64
10329 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10330 AssertRCBreak(rc);
10331 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10332 if ( !fLongModeGuest
10333 || !pCtx->cs.Attr.n.u1Long)
10334 {
10335 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10336 }
10337 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10338 * must be identical if the "IA-32e mode guest" VM-entry
10339 * control is 1 and CS.L is 1. No check applies if the
10340 * CPU supports 64 linear-address bits. */
10341
10342 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10343 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10344 AssertRCBreak(rc);
10345 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10346 VMX_IGS_RFLAGS_RESERVED);
10347 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10348 u32Eflags = u64Val;
10349#else
10350 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10351 AssertRCBreak(rc);
10352 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10353 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10354#endif
10355
10356 if ( fLongModeGuest
10357 || ( fUnrestrictedGuest
10358 && !(u32GuestCr0 & X86_CR0_PE)))
10359 {
10360 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10361 }
10362
10363 uint32_t u32EntryInfo;
10364 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10365 AssertRCBreak(rc);
10366 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10367 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10368 {
10369 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10370 }
10371
10372 /*
10373 * 64-bit checks.
10374 */
10375#if HC_ARCH_BITS == 64
10376 if (fLongModeGuest)
10377 {
10378 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10379 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10380 }
10381
10382 if ( !fLongModeGuest
10383 && (u32GuestCr4 & X86_CR4_PCIDE))
10384 {
10385 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10386 }
10387
10388 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10389 * 51:32 beyond the processor's physical-address width are 0. */
10390
10391 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10392 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10393 {
10394 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10395 }
10396
10397 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10398 AssertRCBreak(rc);
10399 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10400
10401 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10402 AssertRCBreak(rc);
10403 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10404#endif
10405
10406 /*
10407 * PERF_GLOBAL MSR.
10408 */
10409 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10410 {
10411 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10412 AssertRCBreak(rc);
10413 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10414 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10415 }
10416
10417 /*
10418 * PAT MSR.
10419 */
10420 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10421 {
10422 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10423 AssertRCBreak(rc);
10424 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10425 for (unsigned i = 0; i < 8; i++)
10426 {
10427 uint8_t u8Val = (u64Val & 0xff);
10428 if ( u8Val != 0 /* UC */
10429 && u8Val != 1 /* WC */
10430 && u8Val != 4 /* WT */
10431 && u8Val != 5 /* WP */
10432 && u8Val != 6 /* WB */
10433 && u8Val != 7 /* UC- */)
10434 {
10435 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10436 }
10437 u64Val >>= 8;
10438 }
10439 }
10440
10441 /*
10442 * EFER MSR.
10443 */
10444 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10445 {
10446 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10447 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10448 AssertRCBreak(rc);
10449 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10450 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10451 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10452 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10453 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10454 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10455 || !(u32GuestCr0 & X86_CR0_PG)
10456 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10457 VMX_IGS_EFER_LMA_LME_MISMATCH);
10458 }
10459
10460 /*
10461 * Segment registers.
10462 */
10463 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10464 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10465 if (!(u32Eflags & X86_EFL_VM))
10466 {
10467 /* CS */
10468 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10469 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10470 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10471 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10472 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10473 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10474 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10475 /* CS cannot be loaded with NULL in protected mode. */
10476 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10477 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10478 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10479 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10480 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10481 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10482 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10483 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10484 else
10485 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10486
10487 /* SS */
10488 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10489 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10490 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10491 if ( !(pCtx->cr0 & X86_CR0_PE)
10492 || pCtx->cs.Attr.n.u4Type == 3)
10493 {
10494 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10495 }
10496 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10497 {
10498 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10499 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10500 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10501 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10502 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10503 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10504 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10505 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10506 }
10507
10508 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10509 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10510 {
10511 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10512 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10513 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10514 || pCtx->ds.Attr.n.u4Type > 11
10515 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10516 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10517 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10518 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10519 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10520 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10521 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10522 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10523 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10524 }
10525 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10526 {
10527 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10528 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10529 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10530 || pCtx->es.Attr.n.u4Type > 11
10531 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10532 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10533 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10534 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10535 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10536 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10537 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10538 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10539 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10540 }
10541 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10542 {
10543 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10544 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10545 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10546 || pCtx->fs.Attr.n.u4Type > 11
10547 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10548 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10549 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10550 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10551 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10552 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10553 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10554 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10555 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10556 }
10557 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10558 {
10559 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10560 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10561 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10562 || pCtx->gs.Attr.n.u4Type > 11
10563 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10564 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10565 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10566 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10567 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10568 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10569 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10570 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10571 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10572 }
10573 /* 64-bit capable CPUs. */
10574#if HC_ARCH_BITS == 64
10575 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10576 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10577 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10578 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10579 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10580 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10581 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10582 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10583 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10584 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10585 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10586#endif
10587 }
10588 else
10589 {
10590 /* V86 mode checks. */
10591 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10592 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10593 {
10594 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10595 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10596 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10597 }
10598 else
10599 {
10600 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10601 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10602 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10603 }
10604
10605 /* CS */
10606 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10607 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10608 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10609 /* SS */
10610 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10611 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10612 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10613 /* DS */
10614 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10615 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10616 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10617 /* ES */
10618 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10619 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10620 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10621 /* FS */
10622 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10623 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10624 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10625 /* GS */
10626 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10627 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10628 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10629 /* 64-bit capable CPUs. */
10630#if HC_ARCH_BITS == 64
10631 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10632 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10633 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10634 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10635 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10636 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10637 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10638 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10639 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10640 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10641 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10642#endif
10643 }
10644
10645 /*
10646 * TR.
10647 */
10648 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10649 /* 64-bit capable CPUs. */
10650#if HC_ARCH_BITS == 64
10651 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10652#endif
10653 if (fLongModeGuest)
10654 {
10655 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10656 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10657 }
10658 else
10659 {
10660 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10661 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10662 VMX_IGS_TR_ATTR_TYPE_INVALID);
10663 }
10664 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10666 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10667 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10668 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10669 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10670 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10671 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10672
10673 /*
10674 * GDTR and IDTR.
10675 */
10676#if HC_ARCH_BITS == 64
10677 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10678 AssertRCBreak(rc);
10679 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10680
10681 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10682 AssertRCBreak(rc);
10683 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10684#endif
10685
10686 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10687 AssertRCBreak(rc);
10688 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10689
10690 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10691 AssertRCBreak(rc);
10692 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10693
10694 /*
10695 * Guest Non-Register State.
10696 */
10697 /* Activity State. */
10698 uint32_t u32ActivityState;
10699 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10700 AssertRCBreak(rc);
10701 HMVMX_CHECK_BREAK( !u32ActivityState
10702 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10703 VMX_IGS_ACTIVITY_STATE_INVALID);
10704 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10705 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10706 uint32_t u32IntrState;
10707 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10708 AssertRCBreak(rc);
10709 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10710 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10711 {
10712 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10713 }
10714
10715 /** @todo Activity state and injecting interrupts. Left as a todo since we
10716 * currently don't use activity states but ACTIVE. */
10717
10718 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10719 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10720
10721 /* Guest interruptibility-state. */
10722 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10723 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10724 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10725 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10726 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10727 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10728 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10729 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10730 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10731 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10732 {
10733 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10734 {
10735 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10736 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10737 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10738 }
10739 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10740 {
10741 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10742 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10743 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10744 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10745 }
10746 }
10747 /** @todo Assumes the processor is not in SMM. */
10748 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10749 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10750 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10751 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10752 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10753 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10754 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10755 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10756 {
10757 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10758 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10759 }
10760
10761 /* Pending debug exceptions. */
10762#if HC_ARCH_BITS == 64
10763 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10764 AssertRCBreak(rc);
10765 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10766 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10767 u32Val = u64Val; /* For pending debug exceptions checks below. */
10768#else
10769 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10770 AssertRCBreak(rc);
10771 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10772 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10773#endif
10774
10775 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10776 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10777 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10778 {
10779 if ( (u32Eflags & X86_EFL_TF)
10780 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10781 {
10782 /* Bit 14 is PendingDebug.BS. */
10783 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10784 }
10785 if ( !(u32Eflags & X86_EFL_TF)
10786 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10787 {
10788 /* Bit 14 is PendingDebug.BS. */
10789 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10790 }
10791 }
10792
10793 /* VMCS link pointer. */
10794 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10795 AssertRCBreak(rc);
10796 if (u64Val != UINT64_C(0xffffffffffffffff))
10797 {
10798 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10799 /** @todo Bits beyond the processor's physical-address width MBZ. */
10800 /** @todo 32-bit located in memory referenced by value of this field (as a
10801 * physical address) must contain the processor's VMCS revision ID. */
10802 /** @todo SMM checks. */
10803 }
10804
10805 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10806 * not using Nested Paging? */
10807 if ( pVM->hm.s.fNestedPaging
10808 && !fLongModeGuest
10809 && CPUMIsGuestInPAEModeEx(pCtx))
10810 {
10811 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10812 AssertRCBreak(rc);
10813 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10814
10815 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10816 AssertRCBreak(rc);
10817 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10818
10819 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10820 AssertRCBreak(rc);
10821 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10822
10823 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10824 AssertRCBreak(rc);
10825 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10826 }
10827
10828 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10829 if (uError == VMX_IGS_ERROR)
10830 uError = VMX_IGS_REASON_NOT_FOUND;
10831 } while (0);
10832
10833 pVCpu->hm.s.u32HMError = uError;
10834 return uError;
10835
10836#undef HMVMX_ERROR_BREAK
10837#undef HMVMX_CHECK_BREAK
10838}
10839
10840/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10841/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10842/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10843
10844/** @name VM-exit handlers.
10845 * @{
10846 */
10847
10848/**
10849 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10850 */
10851HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10852{
10853 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10854 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10855 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10856 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10857 return VINF_SUCCESS;
10858 return VINF_EM_RAW_INTERRUPT;
10859}
10860
10861
10862/**
10863 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10864 */
10865HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10866{
10867 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10868 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10869
10870 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10871 AssertRCReturn(rc, rc);
10872
10873 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10874 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10875 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10876 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10877
10878 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10879 {
10880 /*
10881 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10882 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10883 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10884 *
10885 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10886 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10887 */
10888 VMXDispatchHostNmi();
10889 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10890 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10891 return VINF_SUCCESS;
10892 }
10893
10894 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10895 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10896 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10897 { /* likely */ }
10898 else
10899 {
10900 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10901 rcStrictRc1 = VINF_SUCCESS;
10902 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10903 return rcStrictRc1;
10904 }
10905
10906 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10907 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10908 switch (uIntType)
10909 {
10910 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10911 Assert(uVector == X86_XCPT_DB);
10912 RT_FALL_THRU();
10913 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10914 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10915 RT_FALL_THRU();
10916 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10917 {
10918 /*
10919 * If there's any exception caused as a result of event injection, the resulting
10920 * secondary/final execption will be pending, we shall continue guest execution
10921 * after injecting the event. The page-fault case is complicated and we manually
10922 * handle any currently pending event in hmR0VmxExitXcptPF.
10923 */
10924 if (!pVCpu->hm.s.Event.fPending)
10925 { /* likely */ }
10926 else if (uVector != X86_XCPT_PF)
10927 {
10928 rc = VINF_SUCCESS;
10929 break;
10930 }
10931
10932 switch (uVector)
10933 {
10934 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10935 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10936 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10937 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10938 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10939 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10940
10941 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10942 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10943 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10944 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10945 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10946 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10947 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10948 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10949 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10950 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10951 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10952 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10953 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10954 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10955 default:
10956 {
10957 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10958 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10959 {
10960 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10961 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10962 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10963
10964 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10965 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10966 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10967 AssertRCReturn(rc, rc);
10968 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10969 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10970 0 /* GCPtrFaultAddress */);
10971 }
10972 else
10973 {
10974 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10975 pVCpu->hm.s.u32HMError = uVector;
10976 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10977 }
10978 break;
10979 }
10980 }
10981 break;
10982 }
10983
10984 default:
10985 {
10986 pVCpu->hm.s.u32HMError = uExitIntInfo;
10987 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10988 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
10989 break;
10990 }
10991 }
10992 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10993 return rc;
10994}
10995
10996
10997/**
10998 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10999 */
11000HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11001{
11002 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11003
11004 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11005 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11006
11007 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11008 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11009 return VINF_SUCCESS;
11010}
11011
11012
11013/**
11014 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11015 */
11016HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11017{
11018 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11019 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11020 {
11021 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11022 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11023 }
11024
11025 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11026
11027 /*
11028 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11029 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11030 */
11031 uint32_t fIntrState = 0;
11032 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11033 AssertRCReturn(rc, rc);
11034
11035 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11036 if ( fBlockSti
11037 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11038 {
11039 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11040 }
11041
11042 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11043 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11044
11045 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11046 return VINF_SUCCESS;
11047}
11048
11049
11050/**
11051 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11052 */
11053HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11054{
11055 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11056 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11057}
11058
11059
11060/**
11061 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11062 */
11063HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11064{
11065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11066 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11067}
11068
11069
11070/**
11071 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11072 */
11073HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11074{
11075 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11076
11077 /*
11078 * Get the state we need and update the exit history entry.
11079 */
11080 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11081 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11082 AssertRCReturn(rc, rc);
11083
11084 VBOXSTRICTRC rcStrict;
11085 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11086 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11087 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11088 if (!pExitRec)
11089 {
11090 /*
11091 * Regular CPUID instruction execution.
11092 */
11093 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11094 if (rcStrict == VINF_SUCCESS)
11095 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11096 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11097 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11098 {
11099 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11100 rcStrict = VINF_SUCCESS;
11101 }
11102 }
11103 else
11104 {
11105 /*
11106 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11107 */
11108 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11109 AssertRCReturn(rc2, rc2);
11110
11111 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11112 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11113
11114 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11115 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11116
11117 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11118 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11119 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11120 }
11121 return rcStrict;
11122}
11123
11124
11125/**
11126 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11127 */
11128HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11129{
11130 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11131 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11132 AssertRCReturn(rc, rc);
11133
11134 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11135 return VINF_EM_RAW_EMULATE_INSTR;
11136
11137 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11138 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11139}
11140
11141
11142/**
11143 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11144 */
11145HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11146{
11147 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11148 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11149 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11150 AssertRCReturn(rc, rc);
11151
11152 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11153 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11154 {
11155 /* If we get a spurious VM-exit when offsetting is enabled,
11156 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11157 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11158 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11160 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11161 }
11162 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11163 {
11164 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11165 rcStrict = VINF_SUCCESS;
11166 }
11167 return rcStrict;
11168}
11169
11170
11171/**
11172 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11173 */
11174HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11175{
11176 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11177 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11178 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11179 AssertRCReturn(rc, rc);
11180
11181 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11182 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11183 {
11184 /* If we get a spurious VM-exit when offsetting is enabled,
11185 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11186 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11187 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11189 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11190 }
11191 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11192 {
11193 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11194 rcStrict = VINF_SUCCESS;
11195 }
11196 return rcStrict;
11197}
11198
11199
11200/**
11201 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11202 */
11203HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11204{
11205 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11206 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11207 AssertRCReturn(rc, rc);
11208
11209 PVM pVM = pVCpu->CTX_SUFF(pVM);
11210 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11211 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11212 if (RT_LIKELY(rc == VINF_SUCCESS))
11213 {
11214 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11215 Assert(pVmxTransient->cbInstr == 2);
11216 }
11217 else
11218 {
11219 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11220 rc = VERR_EM_INTERPRETER;
11221 }
11222 return rc;
11223}
11224
11225
11226/**
11227 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11228 */
11229HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11230{
11231 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11232
11233 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11234 if (EMAreHypercallInstructionsEnabled(pVCpu))
11235 {
11236 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11237 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11238 AssertRCReturn(rc, rc);
11239
11240 /* Perform the hypercall. */
11241 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11242 if (rcStrict == VINF_SUCCESS)
11243 {
11244 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11245 AssertRCReturn(rc, rc);
11246 }
11247 else
11248 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11249 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11250 || RT_FAILURE(rcStrict));
11251
11252 /* If the hypercall changes anything other than guest's general-purpose registers,
11253 we would need to reload the guest changed bits here before VM-entry. */
11254 }
11255 else
11256 Log4Func(("Hypercalls not enabled\n"));
11257
11258 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11259 if (RT_FAILURE(rcStrict))
11260 {
11261 hmR0VmxSetPendingXcptUD(pVCpu);
11262 rcStrict = VINF_SUCCESS;
11263 }
11264
11265 return rcStrict;
11266}
11267
11268
11269/**
11270 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11271 */
11272HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11273{
11274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11275 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11276
11277 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11278 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11279 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11280 AssertRCReturn(rc, rc);
11281
11282 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11283
11284 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11285 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11286 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11287 {
11288 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11289 rcStrict = VINF_SUCCESS;
11290 }
11291 else
11292 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11293 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11294 return rcStrict;
11295}
11296
11297
11298/**
11299 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11300 */
11301HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11302{
11303 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11304 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11305 AssertRCReturn(rc, rc);
11306
11307 PVM pVM = pVCpu->CTX_SUFF(pVM);
11308 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11309 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11310 if (RT_LIKELY(rc == VINF_SUCCESS))
11311 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11312 else
11313 {
11314 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11315 rc = VERR_EM_INTERPRETER;
11316 }
11317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11318 return rc;
11319}
11320
11321
11322/**
11323 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11324 */
11325HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11326{
11327 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11328 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11329 AssertRCReturn(rc, rc);
11330
11331 PVM pVM = pVCpu->CTX_SUFF(pVM);
11332 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11333 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11334 rc = VBOXSTRICTRC_VAL(rc2);
11335 if (RT_LIKELY( rc == VINF_SUCCESS
11336 || rc == VINF_EM_HALT))
11337 {
11338 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11339 AssertRCReturn(rc3, rc3);
11340
11341 if ( rc == VINF_EM_HALT
11342 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11343 rc = VINF_SUCCESS;
11344 }
11345 else
11346 {
11347 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11348 rc = VERR_EM_INTERPRETER;
11349 }
11350 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11351 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11352 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11353 return rc;
11354}
11355
11356
11357/**
11358 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11359 */
11360HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11361{
11362 /*
11363 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11364 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11365 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11366 * VMX root operation. If we get here, something funny is going on.
11367 *
11368 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11369 */
11370 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11371 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11372 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11373}
11374
11375
11376/**
11377 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11378 */
11379HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11380{
11381 /*
11382 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11383 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11384 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11385 * an SMI. If we get here, something funny is going on.
11386 *
11387 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11388 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11389 */
11390 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11391 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11392 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11393}
11394
11395
11396/**
11397 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11398 */
11399HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11400{
11401 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11403 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11404 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11405}
11406
11407
11408/**
11409 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11410 */
11411HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11412{
11413 /*
11414 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11415 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11416 * See Intel spec. 25.3 "Other Causes of VM-exits".
11417 */
11418 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11419 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11420 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11421}
11422
11423
11424/**
11425 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11426 * VM-exit.
11427 */
11428HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11429{
11430 /*
11431 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11432 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11433 *
11434 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11435 * See Intel spec. "23.8 Restrictions on VMX operation".
11436 */
11437 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11438 return VINF_SUCCESS;
11439}
11440
11441
11442/**
11443 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11444 * VM-exit.
11445 */
11446HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11447{
11448 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11449 return VINF_EM_RESET;
11450}
11451
11452
11453/**
11454 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11455 */
11456HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11457{
11458 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11459 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11460
11461 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11462 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11463 AssertRCReturn(rc, rc);
11464
11465 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11466 rc = VINF_SUCCESS;
11467 else
11468 rc = VINF_EM_HALT;
11469
11470 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11471 if (rc != VINF_SUCCESS)
11472 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11473 return rc;
11474}
11475
11476
11477/**
11478 * VM-exit handler for instructions that result in a \#UD exception delivered to
11479 * the guest.
11480 */
11481HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11482{
11483 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11484 hmR0VmxSetPendingXcptUD(pVCpu);
11485 return VINF_SUCCESS;
11486}
11487
11488
11489/**
11490 * VM-exit handler for expiry of the VMX preemption timer.
11491 */
11492HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11493{
11494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11495
11496 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11497 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11498
11499 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11500 PVM pVM = pVCpu->CTX_SUFF(pVM);
11501 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11502 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11503 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11504}
11505
11506
11507/**
11508 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11509 */
11510HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11511{
11512 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11513
11514 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11515 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11516 AssertRCReturn(rc, rc);
11517
11518 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11519 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11520 : HM_CHANGED_XCPT_RAISED_MASK);
11521
11522 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11523 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11524
11525 return rcStrict;
11526}
11527
11528
11529/**
11530 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11531 */
11532HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11533{
11534 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11535 /** @todo Use VM-exit instruction information. */
11536 return VERR_EM_INTERPRETER;
11537}
11538
11539
11540/**
11541 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11542 * Error VM-exit.
11543 */
11544HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11545{
11546 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11547 AssertRCReturn(rc, rc);
11548 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11549 if (RT_FAILURE(rc))
11550 return rc;
11551
11552 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11553 NOREF(uInvalidReason);
11554
11555#ifdef VBOX_STRICT
11556 uint32_t fIntrState;
11557 RTHCUINTREG uHCReg;
11558 uint64_t u64Val;
11559 uint32_t u32Val;
11560
11561 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11562 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11563 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11564 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11565 AssertRCReturn(rc, rc);
11566
11567 Log4(("uInvalidReason %u\n", uInvalidReason));
11568 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11569 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11570 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11571 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11572
11573 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11574 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11575 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11576 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11577 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11578 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11579 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11580 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11581 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11582 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11583 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11584 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11585
11586 hmR0DumpRegs(pVCpu);
11587#else
11588 NOREF(pVmxTransient);
11589#endif
11590
11591 return VERR_VMX_INVALID_GUEST_STATE;
11592}
11593
11594
11595/**
11596 * VM-exit handler for VM-entry failure due to an MSR-load
11597 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11598 */
11599HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11600{
11601 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11602 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11603}
11604
11605
11606/**
11607 * VM-exit handler for VM-entry failure due to a machine-check event
11608 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11609 */
11610HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11611{
11612 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11613 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11614}
11615
11616
11617/**
11618 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11619 * theory.
11620 */
11621HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11622{
11623 RT_NOREF2(pVCpu, pVmxTransient);
11624 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11625 return VERR_VMX_UNDEFINED_EXIT_CODE;
11626}
11627
11628
11629/**
11630 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11631 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11632 * Conditional VM-exit.
11633 */
11634HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11635{
11636 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11637
11638 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11639 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11640 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11641 return VERR_EM_INTERPRETER;
11642 AssertMsgFailed(("Unexpected XDTR access\n"));
11643 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11644}
11645
11646
11647/**
11648 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11649 */
11650HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11651{
11652 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11653
11654 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11655 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11656 return VERR_EM_INTERPRETER;
11657 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11658 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11659}
11660
11661
11662/**
11663 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11664 */
11665HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11666{
11667 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11668
11669 /** @todo Optimize this: We currently drag in in the whole MSR state
11670 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11671 * MSRs required. That would require changes to IEM and possibly CPUM too.
11672 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11673 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11674 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11675 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11676 AssertRCReturn(rc, rc);
11677
11678 Log4Func(("ecx=%#RX32\n", idMsr));
11679
11680#ifdef VBOX_STRICT
11681 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11682 {
11683 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11684 && idMsr != MSR_K6_EFER)
11685 {
11686 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11687 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11688 }
11689 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11690 {
11691 VMXMSREXITREAD enmRead;
11692 VMXMSREXITWRITE enmWrite;
11693 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11694 AssertRCReturn(rc2, rc2);
11695 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11696 {
11697 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11698 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11699 }
11700 }
11701 }
11702#endif
11703
11704 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11705 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11706 if (rcStrict == VINF_SUCCESS)
11707 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11708 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11709 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11710 {
11711 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11712 rcStrict = VINF_SUCCESS;
11713 }
11714 else
11715 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11716
11717 return rcStrict;
11718}
11719
11720
11721/**
11722 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11723 */
11724HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11725{
11726 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11727
11728 /** @todo Optimize this: We currently drag in in the whole MSR state
11729 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11730 * MSRs required. That would require changes to IEM and possibly CPUM too.
11731 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11732 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11733 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11734 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11735 AssertRCReturn(rc, rc);
11736
11737 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11738
11739 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11740 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11741
11742 if (rcStrict == VINF_SUCCESS)
11743 {
11744 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11745
11746 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11747 if ( idMsr == MSR_IA32_APICBASE
11748 || ( idMsr >= MSR_IA32_X2APIC_START
11749 && idMsr <= MSR_IA32_X2APIC_END))
11750 {
11751 /*
11752 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11753 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11754 */
11755 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11756 }
11757 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11758 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11759 else if (idMsr == MSR_K6_EFER)
11760 {
11761 /*
11762 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11763 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11764 * the other bits as well, SCE and NXE. See @bugref{7368}.
11765 */
11766 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11767 | HM_CHANGED_VMX_EXIT_CTLS);
11768 }
11769
11770 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11771 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11772 {
11773 switch (idMsr)
11774 {
11775 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11776 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11777 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11778 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11779 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11780 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11781 default:
11782 {
11783 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11784 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11785 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11786 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11787 break;
11788 }
11789 }
11790 }
11791#ifdef VBOX_STRICT
11792 else
11793 {
11794 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11795 switch (idMsr)
11796 {
11797 case MSR_IA32_SYSENTER_CS:
11798 case MSR_IA32_SYSENTER_EIP:
11799 case MSR_IA32_SYSENTER_ESP:
11800 case MSR_K8_FS_BASE:
11801 case MSR_K8_GS_BASE:
11802 {
11803 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11804 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11805 }
11806
11807 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11808 default:
11809 {
11810 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11811 {
11812 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11813 if (idMsr != MSR_K6_EFER)
11814 {
11815 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11816 idMsr));
11817 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11818 }
11819 }
11820
11821 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11822 {
11823 VMXMSREXITREAD enmRead;
11824 VMXMSREXITWRITE enmWrite;
11825 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11826 AssertRCReturn(rc2, rc2);
11827 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11828 {
11829 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11830 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11831 }
11832 }
11833 break;
11834 }
11835 }
11836 }
11837#endif /* VBOX_STRICT */
11838 }
11839 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11840 {
11841 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11842 rcStrict = VINF_SUCCESS;
11843 }
11844 else
11845 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11846
11847 return rcStrict;
11848}
11849
11850
11851/**
11852 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11853 */
11854HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11855{
11856 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11857 /** @todo The guest has likely hit a contended spinlock. We might want to
11858 * poke a schedule different guest VCPU. */
11859 return VINF_EM_RAW_INTERRUPT;
11860}
11861
11862
11863/**
11864 * VM-exit handler for when the TPR value is lowered below the specified
11865 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11866 */
11867HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11868{
11869 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11870 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11871
11872 /*
11873 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11874 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11875 */
11876 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11877 return VINF_SUCCESS;
11878}
11879
11880
11881/**
11882 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11883 * VM-exit.
11884 *
11885 * @retval VINF_SUCCESS when guest execution can continue.
11886 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11887 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11888 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11889 * interpreter.
11890 */
11891HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11892{
11893 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11894 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11895
11896 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11897 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11898 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11899 AssertRCReturn(rc, rc);
11900
11901 VBOXSTRICTRC rcStrict;
11902 PVM pVM = pVCpu->CTX_SUFF(pVM);
11903 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11904 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11905 switch (uAccessType)
11906 {
11907 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11908 {
11909 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11910 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11911 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11912 AssertMsg( rcStrict == VINF_SUCCESS
11913 || rcStrict == VINF_IEM_RAISED_XCPT
11914 || rcStrict == VINF_PGM_CHANGE_MODE
11915 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11916
11917 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11918 {
11919 case 0:
11920 {
11921 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11922 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11923 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11924 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11925 break;
11926 }
11927
11928 case 2:
11929 {
11930 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11931 /* Nothing to do here, CR2 it's not part of the VMCS. */
11932 break;
11933 }
11934
11935 case 3:
11936 {
11937 Assert( !pVM->hm.s.fNestedPaging
11938 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11939 || pVCpu->hm.s.fUsingDebugLoop);
11940 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11941 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11942 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11943 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11944 break;
11945 }
11946
11947 case 4:
11948 {
11949 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11950 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11951 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11952 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11953 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11954 break;
11955 }
11956
11957 case 8:
11958 {
11959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11960 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11961 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11962 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11963 break;
11964 }
11965 default:
11966 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11967 break;
11968 }
11969 break;
11970 }
11971
11972 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11973 {
11974 Assert( !pVM->hm.s.fNestedPaging
11975 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11976 || pVCpu->hm.s.fUsingDebugLoop
11977 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11978 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11979 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11980 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
11981
11982 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11983 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11984 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
11985 AssertMsg( rcStrict == VINF_SUCCESS
11986 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11987#ifdef VBOX_WITH_STATISTICS
11988 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11989 {
11990 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
11991 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
11992 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
11993 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
11994 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
11995 }
11996#endif
11997 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11998 VBOXSTRICTRC_VAL(rcStrict)));
11999 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12000 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12001 else
12002 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12003 break;
12004 }
12005
12006 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12007 {
12008 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12009 AssertMsg( rcStrict == VINF_SUCCESS
12010 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12011
12012 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12013 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12014 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12015 break;
12016 }
12017
12018 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12019 {
12020 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12021 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12022 AssertMsg( rcStrict == VINF_SUCCESS
12023 || rcStrict == VINF_IEM_RAISED_XCPT
12024 || rcStrict == VINF_PGM_CHANGE_MODE,
12025 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12026
12027 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12029 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12030 break;
12031 }
12032
12033 default:
12034 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12035 VERR_VMX_UNEXPECTED_EXCEPTION);
12036 }
12037
12038 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12039 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12040 if (rcStrict == VINF_IEM_RAISED_XCPT)
12041 {
12042 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
12043 rcStrict = VINF_SUCCESS;
12044 }
12045
12046 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12047 NOREF(pVM);
12048 return rcStrict;
12049}
12050
12051
12052/**
12053 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12054 * VM-exit.
12055 */
12056HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12057{
12058 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12059 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12060
12061 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12062 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12063 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12064 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12065 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12066 AssertRCReturn(rc, rc);
12067
12068 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12069 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12070 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12071 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12072 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12073 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12074 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12075 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12076 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12077
12078 /*
12079 * Update exit history to see if this exit can be optimized.
12080 */
12081 VBOXSTRICTRC rcStrict;
12082 PCEMEXITREC pExitRec = NULL;
12083 if ( !fGstStepping
12084 && !fDbgStepping)
12085 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12086 !fIOString
12087 ? !fIOWrite
12088 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12089 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12090 : !fIOWrite
12091 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12092 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12093 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12094 if (!pExitRec)
12095 {
12096 /* I/O operation lookup arrays. */
12097 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12098 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12099 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12100 uint32_t const cbInstr = pVmxTransient->cbInstr;
12101 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12102 PVM pVM = pVCpu->CTX_SUFF(pVM);
12103 if (fIOString)
12104 {
12105 /*
12106 * INS/OUTS - I/O String instruction.
12107 *
12108 * Use instruction-information if available, otherwise fall back on
12109 * interpreting the instruction.
12110 */
12111 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12112 fIOWrite ? 'w' : 'r'));
12113 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12114 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12115 {
12116 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12117 AssertRCReturn(rc2, rc2);
12118 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12119 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12120 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12121 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12122 if (fIOWrite)
12123 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12124 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12125 else
12126 {
12127 /*
12128 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12129 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12130 * See Intel Instruction spec. for "INS".
12131 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12132 */
12133 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12134 }
12135 }
12136 else
12137 rcStrict = IEMExecOne(pVCpu);
12138
12139 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12140 fUpdateRipAlready = true;
12141 }
12142 else
12143 {
12144 /*
12145 * IN/OUT - I/O instruction.
12146 */
12147 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue,
12148 fIOWrite ? 'w' : 'r'));
12149 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12150 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12151 if (fIOWrite)
12152 {
12153 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12154 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12155 }
12156 else
12157 {
12158 uint32_t u32Result = 0;
12159 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12160 if (IOM_SUCCESS(rcStrict))
12161 {
12162 /* Save result of I/O IN instr. in AL/AX/EAX. */
12163 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12164 }
12165 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12166 HMR0SavePendingIOPortRead(pVCpu, pCtx->rip, pCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12167 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12168 }
12169 }
12170
12171 if (IOM_SUCCESS(rcStrict))
12172 {
12173 if (!fUpdateRipAlready)
12174 {
12175 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12176 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12177 }
12178
12179 /*
12180 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12181 * while booting Fedora 17 64-bit guest.
12182 *
12183 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12184 */
12185 if (fIOString)
12186 {
12187 /** @todo Single-step for INS/OUTS with REP prefix? */
12188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12189 }
12190 else if ( !fDbgStepping
12191 && fGstStepping)
12192 {
12193 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12194 AssertRCReturn(rc, rc);
12195 }
12196
12197 /*
12198 * If any I/O breakpoints are armed, we need to check if one triggered
12199 * and take appropriate action.
12200 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12201 */
12202 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12203 AssertRCReturn(rc, rc);
12204
12205 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12206 * execution engines about whether hyper BPs and such are pending. */
12207 uint32_t const uDr7 = pCtx->dr[7];
12208 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12209 && X86_DR7_ANY_RW_IO(uDr7)
12210 && (pCtx->cr4 & X86_CR4_DE))
12211 || DBGFBpIsHwIoArmed(pVM)))
12212 {
12213 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12214
12215 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12216 VMMRZCallRing3Disable(pVCpu);
12217 HM_DISABLE_PREEMPT();
12218
12219 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12220
12221 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12222 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12223 {
12224 /* Raise #DB. */
12225 if (fIsGuestDbgActive)
12226 ASMSetDR6(pCtx->dr[6]);
12227 if (pCtx->dr[7] != uDr7)
12228 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12229
12230 hmR0VmxSetPendingXcptDB(pVCpu);
12231 }
12232 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12233 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12234 else if ( rcStrict2 != VINF_SUCCESS
12235 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12236 rcStrict = rcStrict2;
12237 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12238
12239 HM_RESTORE_PREEMPT();
12240 VMMRZCallRing3Enable(pVCpu);
12241 }
12242 }
12243
12244#ifdef VBOX_STRICT
12245 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12246 Assert(!fIOWrite);
12247 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12248 Assert(fIOWrite);
12249 else
12250 {
12251# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12252 * statuses, that the VMM device and some others may return. See
12253 * IOM_SUCCESS() for guidance. */
12254 AssertMsg( RT_FAILURE(rcStrict)
12255 || rcStrict == VINF_SUCCESS
12256 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12257 || rcStrict == VINF_EM_DBG_BREAKPOINT
12258 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12259 || rcStrict == VINF_EM_RAW_TO_R3
12260 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12261# endif
12262 }
12263#endif
12264 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12265 }
12266 else
12267 {
12268 /*
12269 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12270 */
12271 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12272 AssertRCReturn(rc2, rc2);
12273 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12274 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12275 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12276 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12277 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12278 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12279
12280 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12281 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12282
12283 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12284 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12285 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12286 }
12287 return rcStrict;
12288}
12289
12290
12291/**
12292 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12293 * VM-exit.
12294 */
12295HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12296{
12297 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12298
12299 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12300 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12301 AssertRCReturn(rc, rc);
12302 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12303 {
12304 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12305 AssertRCReturn(rc, rc);
12306 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12307 {
12308 uint32_t uErrCode;
12309 RTGCUINTPTR GCPtrFaultAddress;
12310 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12311 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12312 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12313 if (fErrorCodeValid)
12314 {
12315 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12316 AssertRCReturn(rc, rc);
12317 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12318 }
12319 else
12320 uErrCode = 0;
12321
12322 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12323 && uVector == X86_XCPT_PF)
12324 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12325 else
12326 GCPtrFaultAddress = 0;
12327
12328 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12329 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12330
12331 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12333 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12334 }
12335 }
12336
12337 /* Fall back to the interpreter to emulate the task-switch. */
12338 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12339 return VERR_EM_INTERPRETER;
12340}
12341
12342
12343/**
12344 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12345 */
12346HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12347{
12348 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12349 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12350 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12352 AssertRCReturn(rc, rc);
12353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12354 return VINF_EM_DBG_STEPPED;
12355}
12356
12357
12358/**
12359 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12360 */
12361HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12362{
12363 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12364
12365 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12366
12367 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12368 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12369 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12370 {
12371 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12372 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12373 {
12374 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12375 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12376 }
12377 }
12378 else
12379 {
12380 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12381 rcStrict1 = VINF_SUCCESS;
12382 return rcStrict1;
12383 }
12384
12385 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12386 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12387 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12388 AssertRCReturn(rc, rc);
12389
12390 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12391 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12392 VBOXSTRICTRC rcStrict2;
12393 switch (uAccessType)
12394 {
12395 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12396 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12397 {
12398 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12399 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12400 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12401
12402 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12403 GCPhys &= PAGE_BASE_GC_MASK;
12404 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12405 PVM pVM = pVCpu->CTX_SUFF(pVM);
12406 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12407 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12408
12409 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12410 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12411 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12412 CPUMCTX2CORE(pCtx), GCPhys);
12413 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12414 if ( rcStrict2 == VINF_SUCCESS
12415 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12416 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12417 {
12418 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12419 | HM_CHANGED_GUEST_APIC_TPR);
12420 rcStrict2 = VINF_SUCCESS;
12421 }
12422 break;
12423 }
12424
12425 default:
12426 Log4Func(("uAccessType=%#x\n", uAccessType));
12427 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12428 break;
12429 }
12430
12431 if (rcStrict2 != VINF_SUCCESS)
12432 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12433 return rcStrict2;
12434}
12435
12436
12437/**
12438 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12439 * VM-exit.
12440 */
12441HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12442{
12443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12444
12445 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12446 if (pVmxTransient->fWasGuestDebugStateActive)
12447 {
12448 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12449 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12450 }
12451
12452 if ( !pVCpu->hm.s.fSingleInstruction
12453 && !pVmxTransient->fWasHyperDebugStateActive)
12454 {
12455 Assert(!DBGFIsStepping(pVCpu));
12456 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12457
12458 /* Don't intercept MOV DRx any more. */
12459 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12460 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12461 AssertRCReturn(rc, rc);
12462
12463 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12464 VMMRZCallRing3Disable(pVCpu);
12465 HM_DISABLE_PREEMPT();
12466
12467 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12468 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12469 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12470
12471 HM_RESTORE_PREEMPT();
12472 VMMRZCallRing3Enable(pVCpu);
12473
12474#ifdef VBOX_WITH_STATISTICS
12475 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12476 AssertRCReturn(rc, rc);
12477 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12478 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12479 else
12480 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12481#endif
12482 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12483 return VINF_SUCCESS;
12484 }
12485
12486 /*
12487 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12488 * Update the segment registers and DR7 from the CPU.
12489 */
12490 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12491 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12492 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12493 AssertRCReturn(rc, rc);
12494 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12495
12496 PVM pVM = pVCpu->CTX_SUFF(pVM);
12497 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12498 {
12499 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12500 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12501 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12502 if (RT_SUCCESS(rc))
12503 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12504 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12505 }
12506 else
12507 {
12508 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12509 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12510 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12511 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12512 }
12513
12514 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12515 if (RT_SUCCESS(rc))
12516 {
12517 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12518 AssertRCReturn(rc2, rc2);
12519 return VINF_SUCCESS;
12520 }
12521 return rc;
12522}
12523
12524
12525/**
12526 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12527 * Conditional VM-exit.
12528 */
12529HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12530{
12531 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12532 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12533
12534 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12535 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12536 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12537 {
12538 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12539 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12540 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12541 {
12542 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12543 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12544 }
12545 }
12546 else
12547 {
12548 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12549 rcStrict1 = VINF_SUCCESS;
12550 return rcStrict1;
12551 }
12552
12553 /*
12554 * Get sufficent state and update the exit history entry.
12555 */
12556 RTGCPHYS GCPhys;
12557 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12558 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12559 AssertRCReturn(rc, rc);
12560
12561 VBOXSTRICTRC rcStrict;
12562 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12563 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12564 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12565 if (!pExitRec)
12566 {
12567 /*
12568 * If we succeed, resume guest execution.
12569 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12570 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12571 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12572 * weird case. See @bugref{6043}.
12573 */
12574 PVM pVM = pVCpu->CTX_SUFF(pVM);
12575 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12576 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12577 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12578 if ( rcStrict == VINF_SUCCESS
12579 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12580 || rcStrict == VERR_PAGE_NOT_PRESENT)
12581 {
12582 /* Successfully handled MMIO operation. */
12583 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12584 | HM_CHANGED_GUEST_APIC_TPR);
12585 rcStrict = VINF_SUCCESS;
12586 }
12587 }
12588 else
12589 {
12590 /*
12591 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12592 */
12593 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12594 AssertRCReturn(rc2, rc2);
12595
12596 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12597 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12598
12599 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12600 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12601
12602 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12603 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12604 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12605 }
12606 return VBOXSTRICTRC_TODO(rcStrict);
12607}
12608
12609
12610/**
12611 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12612 * VM-exit.
12613 */
12614HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12615{
12616 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12617 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12618
12619 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12620 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12621 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12622 {
12623 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12624 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12625 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12626 }
12627 else
12628 {
12629 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12630 rcStrict1 = VINF_SUCCESS;
12631 return rcStrict1;
12632 }
12633
12634 RTGCPHYS GCPhys;
12635 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12636 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12637 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12638 AssertRCReturn(rc, rc);
12639
12640 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12641 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12642
12643 RTGCUINT uErrorCode = 0;
12644 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12645 uErrorCode |= X86_TRAP_PF_ID;
12646 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12647 uErrorCode |= X86_TRAP_PF_RW;
12648 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12649 uErrorCode |= X86_TRAP_PF_P;
12650
12651 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12652
12653
12654 /* Handle the pagefault trap for the nested shadow table. */
12655 PVM pVM = pVCpu->CTX_SUFF(pVM);
12656 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12657
12658 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12659 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12660
12661 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12662 TRPMResetTrap(pVCpu);
12663
12664 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12665 if ( rcStrict2 == VINF_SUCCESS
12666 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12667 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12668 {
12669 /* Successfully synced our nested page tables. */
12670 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12671 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12672 return VINF_SUCCESS;
12673 }
12674
12675 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12676 return rcStrict2;
12677}
12678
12679/** @} */
12680
12681/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12682/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12683/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12684
12685/** @name VM-exit exception handlers.
12686 * @{
12687 */
12688
12689/**
12690 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12691 */
12692static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12693{
12694 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12695 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12696
12697 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12698 AssertRCReturn(rc, rc);
12699
12700 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12701 {
12702 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12703 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12704
12705 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12706 * provides VM-exit instruction length. If this causes problem later,
12707 * disassemble the instruction like it's done on AMD-V. */
12708 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12709 AssertRCReturn(rc2, rc2);
12710 return rc;
12711 }
12712
12713 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12714 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12715 return rc;
12716}
12717
12718
12719/**
12720 * VM-exit exception handler for \#BP (Breakpoint exception).
12721 */
12722static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12723{
12724 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12725 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12726
12727 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12728 AssertRCReturn(rc, rc);
12729
12730 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12731 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12732 if (rc == VINF_EM_RAW_GUEST_TRAP)
12733 {
12734 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12735 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12736 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12737 AssertRCReturn(rc, rc);
12738
12739 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12740 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12741 }
12742
12743 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12744 return rc;
12745}
12746
12747
12748/**
12749 * VM-exit exception handler for \#AC (alignment check exception).
12750 */
12751static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12752{
12753 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12754
12755 /*
12756 * Re-inject it. We'll detect any nesting before getting here.
12757 */
12758 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12759 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12760 AssertRCReturn(rc, rc);
12761 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12762
12763 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12764 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12765 return VINF_SUCCESS;
12766}
12767
12768
12769/**
12770 * VM-exit exception handler for \#DB (Debug exception).
12771 */
12772static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12773{
12774 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12775 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12776
12777 /*
12778 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12779 * for processing.
12780 */
12781 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12782
12783 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12784 uint64_t uDR6 = X86_DR6_INIT_VAL;
12785 uDR6 |= ( pVmxTransient->uExitQualification
12786 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12787
12788 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12789 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12790 Log6Func(("rc=%Rrc\n", rc));
12791 if (rc == VINF_EM_RAW_GUEST_TRAP)
12792 {
12793 /*
12794 * The exception was for the guest. Update DR6, DR7.GD and
12795 * IA32_DEBUGCTL.LBR before forwarding it.
12796 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12797 */
12798 VMMRZCallRing3Disable(pVCpu);
12799 HM_DISABLE_PREEMPT();
12800
12801 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12802 pCtx->dr[6] |= uDR6;
12803 if (CPUMIsGuestDebugStateActive(pVCpu))
12804 ASMSetDR6(pCtx->dr[6]);
12805
12806 HM_RESTORE_PREEMPT();
12807 VMMRZCallRing3Enable(pVCpu);
12808
12809 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12810 AssertRCReturn(rc, rc);
12811
12812 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12813 pCtx->dr[7] &= ~X86_DR7_GD;
12814
12815 /* Paranoia. */
12816 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12817 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12818
12819 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12820 AssertRCReturn(rc, rc);
12821
12822 /*
12823 * Raise #DB in the guest.
12824 *
12825 * It is important to reflect exactly what the VM-exit gave us (preserving the
12826 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12827 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12828 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12829 *
12830 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12831 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12832 */
12833 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12834 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12835 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12836 AssertRCReturn(rc, rc);
12837 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12838 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12839 return VINF_SUCCESS;
12840 }
12841
12842 /*
12843 * Not a guest trap, must be a hypervisor related debug event then.
12844 * Update DR6 in case someone is interested in it.
12845 */
12846 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12847 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12848 CPUMSetHyperDR6(pVCpu, uDR6);
12849
12850 return rc;
12851}
12852
12853/**
12854 * VM-exit exception handler for \#GP (General-protection exception).
12855 *
12856 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12857 */
12858static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12859{
12860 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12861 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12862
12863 int rc;
12864 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12865 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12866 { /* likely */ }
12867 else
12868 {
12869#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12870 Assert(pVCpu->hm.s.fUsingDebugLoop);
12871#endif
12872 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12873 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12874 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12875 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12876 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12877 AssertRCReturn(rc, rc);
12878 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12879 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12880 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12881 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12882 return rc;
12883 }
12884
12885 Assert(CPUMIsGuestInRealModeEx(pCtx));
12886 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12887
12888 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12889 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12890 AssertRCReturn(rc, rc);
12891
12892 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12893 uint32_t cbOp = 0;
12894 PVM pVM = pVCpu->CTX_SUFF(pVM);
12895 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12896 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12897 if (RT_SUCCESS(rc))
12898 {
12899 rc = VINF_SUCCESS;
12900 Assert(cbOp == pDis->cbInstr);
12901 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12902 switch (pDis->pCurInstr->uOpcode)
12903 {
12904 case OP_CLI:
12905 {
12906 pCtx->eflags.Bits.u1IF = 0;
12907 pCtx->eflags.Bits.u1RF = 0;
12908 pCtx->rip += pDis->cbInstr;
12909 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12910 if ( !fDbgStepping
12911 && pCtx->eflags.Bits.u1TF)
12912 {
12913 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12914 AssertRCReturn(rc, rc);
12915 }
12916 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12917 break;
12918 }
12919
12920 case OP_STI:
12921 {
12922 bool fOldIF = pCtx->eflags.Bits.u1IF;
12923 pCtx->eflags.Bits.u1IF = 1;
12924 pCtx->eflags.Bits.u1RF = 0;
12925 pCtx->rip += pDis->cbInstr;
12926 if (!fOldIF)
12927 {
12928 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12929 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12930 }
12931 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12932 if ( !fDbgStepping
12933 && pCtx->eflags.Bits.u1TF)
12934 {
12935 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12936 AssertRCReturn(rc, rc);
12937 }
12938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12939 break;
12940 }
12941
12942 case OP_HLT:
12943 {
12944 rc = VINF_EM_HALT;
12945 pCtx->rip += pDis->cbInstr;
12946 pCtx->eflags.Bits.u1RF = 0;
12947 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12948 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12949 break;
12950 }
12951
12952 case OP_POPF:
12953 {
12954 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12955 uint32_t cbParm;
12956 uint32_t uMask;
12957 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12958 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12959 {
12960 cbParm = 4;
12961 uMask = 0xffffffff;
12962 }
12963 else
12964 {
12965 cbParm = 2;
12966 uMask = 0xffff;
12967 }
12968
12969 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12970 RTGCPTR GCPtrStack = 0;
12971 X86EFLAGS Eflags;
12972 Eflags.u32 = 0;
12973 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12974 &GCPtrStack);
12975 if (RT_SUCCESS(rc))
12976 {
12977 Assert(sizeof(Eflags.u32) >= cbParm);
12978 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
12979 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
12980 }
12981 if (RT_FAILURE(rc))
12982 {
12983 rc = VERR_EM_INTERPRETER;
12984 break;
12985 }
12986 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
12987 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
12988 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
12989 pCtx->esp += cbParm;
12990 pCtx->esp &= uMask;
12991 pCtx->rip += pDis->cbInstr;
12992 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12993 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
12994 POPF restores EFLAGS.TF. */
12995 if ( !fDbgStepping
12996 && fGstStepping)
12997 {
12998 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12999 AssertRCReturn(rc, rc);
13000 }
13001 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13002 break;
13003 }
13004
13005 case OP_PUSHF:
13006 {
13007 uint32_t cbParm;
13008 uint32_t uMask;
13009 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13010 {
13011 cbParm = 4;
13012 uMask = 0xffffffff;
13013 }
13014 else
13015 {
13016 cbParm = 2;
13017 uMask = 0xffff;
13018 }
13019
13020 /* Get the stack pointer & push the contents of eflags onto the stack. */
13021 RTGCPTR GCPtrStack = 0;
13022 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13023 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13024 if (RT_FAILURE(rc))
13025 {
13026 rc = VERR_EM_INTERPRETER;
13027 break;
13028 }
13029 X86EFLAGS Eflags = pCtx->eflags;
13030 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13031 Eflags.Bits.u1RF = 0;
13032 Eflags.Bits.u1VM = 0;
13033
13034 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13035 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13036 {
13037 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13038 rc = VERR_EM_INTERPRETER;
13039 break;
13040 }
13041 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13042 pCtx->esp -= cbParm;
13043 pCtx->esp &= uMask;
13044 pCtx->rip += pDis->cbInstr;
13045 pCtx->eflags.Bits.u1RF = 0;
13046 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13047 if ( !fDbgStepping
13048 && pCtx->eflags.Bits.u1TF)
13049 {
13050 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13051 AssertRCReturn(rc, rc);
13052 }
13053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13054 break;
13055 }
13056
13057 case OP_IRET:
13058 {
13059 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13060 * instruction reference. */
13061 RTGCPTR GCPtrStack = 0;
13062 uint32_t uMask = 0xffff;
13063 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13064 uint16_t aIretFrame[3];
13065 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13066 {
13067 rc = VERR_EM_INTERPRETER;
13068 break;
13069 }
13070 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13071 &GCPtrStack);
13072 if (RT_SUCCESS(rc))
13073 {
13074 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13075 PGMACCESSORIGIN_HM));
13076 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13077 }
13078 if (RT_FAILURE(rc))
13079 {
13080 rc = VERR_EM_INTERPRETER;
13081 break;
13082 }
13083 pCtx->eip = 0;
13084 pCtx->ip = aIretFrame[0];
13085 pCtx->cs.Sel = aIretFrame[1];
13086 pCtx->cs.ValidSel = aIretFrame[1];
13087 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13088 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13089 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13090 pCtx->sp += sizeof(aIretFrame);
13091 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13092 | HM_CHANGED_GUEST_CS);
13093 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13094 if ( !fDbgStepping
13095 && fGstStepping)
13096 {
13097 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13098 AssertRCReturn(rc, rc);
13099 }
13100 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13101 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13102 break;
13103 }
13104
13105 case OP_INT:
13106 {
13107 uint16_t uVector = pDis->Param1.uValue & 0xff;
13108 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13109 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13110 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13111 break;
13112 }
13113
13114 case OP_INTO:
13115 {
13116 if (pCtx->eflags.Bits.u1OF)
13117 {
13118 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13119 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13121 }
13122 else
13123 {
13124 pCtx->eflags.Bits.u1RF = 0;
13125 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13126 }
13127 break;
13128 }
13129
13130 default:
13131 {
13132 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13133 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13134 EMCODETYPE_SUPERVISOR);
13135 rc = VBOXSTRICTRC_VAL(rc2);
13136 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13137 /** @todo We have to set pending-debug exceptions here when the guest is
13138 * single-stepping depending on the instruction that was interpreted. */
13139 Log4Func(("#GP rc=%Rrc\n", rc));
13140 break;
13141 }
13142 }
13143 }
13144 else
13145 rc = VERR_EM_INTERPRETER;
13146
13147 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13148 ("#GP Unexpected rc=%Rrc\n", rc));
13149 return rc;
13150}
13151
13152
13153/**
13154 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13155 * the exception reported in the VMX transient structure back into the VM.
13156 *
13157 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13158 * up-to-date.
13159 */
13160static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13161{
13162 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13163#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13164 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13165 ("uVector=%#x u32XcptBitmap=%#X32\n",
13166 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13167#endif
13168
13169 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13170 hmR0VmxCheckExitDueToEventDelivery(). */
13171 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13172 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13173 AssertRCReturn(rc, rc);
13174 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13175
13176#ifdef DEBUG_ramshankar
13177 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13178 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13179 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13180#endif
13181
13182 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13183 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13184 return VINF_SUCCESS;
13185}
13186
13187
13188/**
13189 * VM-exit exception handler for \#PF (Page-fault exception).
13190 */
13191static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13192{
13193 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13194 PVM pVM = pVCpu->CTX_SUFF(pVM);
13195 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13196 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13197 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13198 AssertRCReturn(rc, rc);
13199
13200 if (!pVM->hm.s.fNestedPaging)
13201 { /* likely */ }
13202 else
13203 {
13204#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13205 Assert(pVCpu->hm.s.fUsingDebugLoop);
13206#endif
13207 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13208 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13209 {
13210 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13211 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13212 }
13213 else
13214 {
13215 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13216 hmR0VmxSetPendingXcptDF(pVCpu);
13217 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13218 }
13219 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13220 return rc;
13221 }
13222
13223 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13224 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13225 if (pVmxTransient->fVectoringPF)
13226 {
13227 Assert(pVCpu->hm.s.Event.fPending);
13228 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13229 }
13230
13231 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13232 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13233 AssertRCReturn(rc, rc);
13234
13235 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13236 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13237
13238 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13239 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13240 (RTGCPTR)pVmxTransient->uExitQualification);
13241
13242 Log4Func(("#PF: rc=%Rrc\n", rc));
13243 if (rc == VINF_SUCCESS)
13244 {
13245 /*
13246 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13247 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13248 */
13249 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13250 TRPMResetTrap(pVCpu);
13251 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13252 return rc;
13253 }
13254
13255 if (rc == VINF_EM_RAW_GUEST_TRAP)
13256 {
13257 if (!pVmxTransient->fVectoringDoublePF)
13258 {
13259 /* It's a guest page fault and needs to be reflected to the guest. */
13260 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13261 TRPMResetTrap(pVCpu);
13262 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13263 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13264 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13265 }
13266 else
13267 {
13268 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13269 TRPMResetTrap(pVCpu);
13270 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13271 hmR0VmxSetPendingXcptDF(pVCpu);
13272 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13273 }
13274
13275 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13276 return VINF_SUCCESS;
13277 }
13278
13279 TRPMResetTrap(pVCpu);
13280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13281 return rc;
13282}
13283
13284/** @} */
13285
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette