VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 73437

Last change on this file since 73437 was 73437, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Nested VMX: bugref:9180 Fix bug while injecting double-fault exception.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 562.4 KB
Line 
1/* $Id: HMVMXR0.cpp 73437 2018-08-02 06:11:26Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu) Assert( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE(a_pVCpu) AssertMsg( VMMR0ThreadCtxHookIsEnabled((a_pVCpu)) \
152 || (a_pVCpu)->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 (a_pVCpu)->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pVmxTransient Pointer to the VMX-transient structure.
335 */
336#ifndef HMVMX_USE_FUNCTION_TABLE
337typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
338#else
339typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
340/** Pointer to VM-exit handler. */
341typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
342#endif
343
344/**
345 * VMX VM-exit handler, non-strict status code.
346 *
347 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
348 *
349 * @returns VBox status code, no informational status code returned.
350 * @param pVCpu The cross context virtual CPU structure.
351 * @param pVmxTransient Pointer to the VMX-transient structure.
352 *
353 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
354 * use of that status code will be replaced with VINF_EM_SOMETHING
355 * later when switching over to IEM.
356 */
357#ifndef HMVMX_USE_FUNCTION_TABLE
358typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
359#else
360typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
361#endif
362
363
364/*********************************************************************************************************************************
365* Internal Functions *
366*********************************************************************************************************************************/
367static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
368static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
369static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
370static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
371static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
372 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
373#if HC_ARCH_BITS == 32
374static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
375#endif
376#ifndef HMVMX_USE_FUNCTION_TABLE
377DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
378# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
379# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
380#else
381# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
382# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
383#endif
384
385
386/** @name VM-exit handlers.
387 * @{
388 */
389static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
390static FNVMXEXITHANDLER hmR0VmxExitExtInt;
391static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
392static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
393static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
394static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
395static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
396static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
398static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
399static FNVMXEXITHANDLER hmR0VmxExitCpuid;
400static FNVMXEXITHANDLER hmR0VmxExitGetsec;
401static FNVMXEXITHANDLER hmR0VmxExitHlt;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
403static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
404static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
405static FNVMXEXITHANDLER hmR0VmxExitVmcall;
406static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
409static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
410static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
411static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
412static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
413static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
417static FNVMXEXITHANDLER hmR0VmxExitMwait;
418static FNVMXEXITHANDLER hmR0VmxExitMtf;
419static FNVMXEXITHANDLER hmR0VmxExitMonitor;
420static FNVMXEXITHANDLER hmR0VmxExitPause;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
423static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
424static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
425static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
426static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
427static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
428static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
429static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
431static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
432static FNVMXEXITHANDLER hmR0VmxExitRdrand;
433static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
434/** @} */
435
436static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
437static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
438static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
439static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
440static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
441static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient);
443static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu);
444
445
446/*********************************************************************************************************************************
447* Global Variables *
448*********************************************************************************************************************************/
449#ifdef HMVMX_USE_FUNCTION_TABLE
450
451/**
452 * VMX_EXIT dispatch table.
453 */
454static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
455{
456 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
457 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
458 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
459 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
460 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
461 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
462 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
463 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
464 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
465 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
466 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
467 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
468 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
469 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
470 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
471 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
472 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
473 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
474 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
475 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
476 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
477 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
478 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
479 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
480 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
481 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
482 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
483 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
484 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
485 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
486 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
487 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
488 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
489 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
490 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
491 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
492 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
493 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
494 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
495 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
496 /* 40 UNDEFINED */ hmR0VmxExitPause,
497 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
498 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
499 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
500 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
501 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
503 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
504 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
505 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
506 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
507 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
508 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
509 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
510 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
511 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
512 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
513 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
514 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
515 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
516 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
517 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
518 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
519 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
520 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
521};
522#endif /* HMVMX_USE_FUNCTION_TABLE */
523
524#ifdef VBOX_STRICT
525static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
526{
527 /* 0 */ "(Not Used)",
528 /* 1 */ "VMCALL executed in VMX root operation.",
529 /* 2 */ "VMCLEAR with invalid physical address.",
530 /* 3 */ "VMCLEAR with VMXON pointer.",
531 /* 4 */ "VMLAUNCH with non-clear VMCS.",
532 /* 5 */ "VMRESUME with non-launched VMCS.",
533 /* 6 */ "VMRESUME after VMXOFF",
534 /* 7 */ "VM-entry with invalid control fields.",
535 /* 8 */ "VM-entry with invalid host state fields.",
536 /* 9 */ "VMPTRLD with invalid physical address.",
537 /* 10 */ "VMPTRLD with VMXON pointer.",
538 /* 11 */ "VMPTRLD with incorrect revision identifier.",
539 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
540 /* 13 */ "VMWRITE to read-only VMCS component.",
541 /* 14 */ "(Not Used)",
542 /* 15 */ "VMXON executed in VMX root operation.",
543 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
544 /* 17 */ "VM-entry with non-launched executing VMCS.",
545 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
546 /* 19 */ "VMCALL with non-clear VMCS.",
547 /* 20 */ "VMCALL with invalid VM-exit control fields.",
548 /* 21 */ "(Not Used)",
549 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
550 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
551 /* 24 */ "VMCALL with invalid SMM-monitor features.",
552 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
553 /* 26 */ "VM-entry with events blocked by MOV SS.",
554 /* 27 */ "(Not Used)",
555 /* 28 */ "Invalid operand to INVEPT/INVVPID."
556};
557#endif /* VBOX_STRICT */
558
559
560
561/**
562 * Updates the VM's last error record.
563 *
564 * If there was a VMX instruction error, reads the error data from the VMCS and
565 * updates VCPU's last error record as well.
566 *
567 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
568 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
569 * VERR_VMX_INVALID_VMCS_FIELD.
570 * @param rc The error code.
571 */
572static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
573{
574 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
575 || rc == VERR_VMX_UNABLE_TO_START_VM)
576 {
577 AssertPtrReturnVoid(pVCpu);
578 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
579 }
580 pVCpu->CTX_SUFF(pVM)->hm.s.rcInit = rc;
581}
582
583
584/**
585 * Reads the VM-entry interruption-information field from the VMCS into the VMX
586 * transient structure.
587 *
588 * @returns VBox status code.
589 * @param pVmxTransient Pointer to the VMX transient structure.
590 *
591 * @remarks No-long-jump zone!!!
592 */
593DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
594{
595 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
596 AssertRCReturn(rc, rc);
597 return VINF_SUCCESS;
598}
599
600#ifdef VBOX_STRICT
601/**
602 * Reads the VM-entry exception error code field from the VMCS into
603 * the VMX transient structure.
604 *
605 * @returns VBox status code.
606 * @param pVmxTransient Pointer to the VMX transient structure.
607 *
608 * @remarks No-long-jump zone!!!
609 */
610DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
611{
612 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
613 AssertRCReturn(rc, rc);
614 return VINF_SUCCESS;
615}
616
617
618/**
619 * Reads the VM-entry exception error code field from the VMCS into
620 * the VMX transient structure.
621 *
622 * @returns VBox status code.
623 * @param pVmxTransient Pointer to the VMX transient structure.
624 *
625 * @remarks No-long-jump zone!!!
626 */
627DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
628{
629 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
630 AssertRCReturn(rc, rc);
631 return VINF_SUCCESS;
632}
633#endif /* VBOX_STRICT */
634
635
636/**
637 * Reads the VM-exit interruption-information field from the VMCS into the VMX
638 * transient structure.
639 *
640 * @returns VBox status code.
641 * @param pVmxTransient Pointer to the VMX transient structure.
642 */
643DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
644{
645 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
646 {
647 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
648 AssertRCReturn(rc,rc);
649 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
650 }
651 return VINF_SUCCESS;
652}
653
654
655/**
656 * Reads the VM-exit interruption error code from the VMCS into the VMX
657 * transient structure.
658 *
659 * @returns VBox status code.
660 * @param pVmxTransient Pointer to the VMX transient structure.
661 */
662DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
663{
664 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
665 {
666 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
667 AssertRCReturn(rc, rc);
668 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
669 }
670 return VINF_SUCCESS;
671}
672
673
674/**
675 * Reads the VM-exit instruction length field from the VMCS into the VMX
676 * transient structure.
677 *
678 * @returns VBox status code.
679 * @param pVmxTransient Pointer to the VMX transient structure.
680 */
681DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
682{
683 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
684 {
685 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
686 AssertRCReturn(rc, rc);
687 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
688 }
689 return VINF_SUCCESS;
690}
691
692
693/**
694 * Reads the VM-exit instruction-information field from the VMCS into
695 * the VMX transient structure.
696 *
697 * @returns VBox status code.
698 * @param pVmxTransient Pointer to the VMX transient structure.
699 */
700DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
701{
702 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
703 {
704 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
705 AssertRCReturn(rc, rc);
706 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
707 }
708 return VINF_SUCCESS;
709}
710
711
712/**
713 * Reads the exit code qualification from the VMCS into the VMX transient
714 * structure.
715 *
716 * @returns VBox status code.
717 * @param pVCpu The cross context virtual CPU structure of the
718 * calling EMT. (Required for the VMCS cache case.)
719 * @param pVmxTransient Pointer to the VMX transient structure.
720 */
721DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
722{
723 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
724 {
725 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
726 AssertRCReturn(rc, rc);
727 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
728 }
729 return VINF_SUCCESS;
730}
731
732
733/**
734 * Reads the IDT-vectoring information field from the VMCS into the VMX
735 * transient structure.
736 *
737 * @returns VBox status code.
738 * @param pVmxTransient Pointer to the VMX transient structure.
739 *
740 * @remarks No-long-jump zone!!!
741 */
742DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
743{
744 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
745 {
746 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
747 AssertRCReturn(rc, rc);
748 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
749 }
750 return VINF_SUCCESS;
751}
752
753
754/**
755 * Reads the IDT-vectoring error code from the VMCS into the VMX
756 * transient structure.
757 *
758 * @returns VBox status code.
759 * @param pVmxTransient Pointer to the VMX transient structure.
760 */
761DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
762{
763 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
764 {
765 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
766 AssertRCReturn(rc, rc);
767 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
768 }
769 return VINF_SUCCESS;
770}
771
772
773/**
774 * Enters VMX root mode operation on the current CPU.
775 *
776 * @returns VBox status code.
777 * @param pVM The cross context VM structure. Can be
778 * NULL, after a resume.
779 * @param HCPhysCpuPage Physical address of the VMXON region.
780 * @param pvCpuPage Pointer to the VMXON region.
781 */
782static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
783{
784 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
785 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
786 Assert(pvCpuPage);
787 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
788
789 if (pVM)
790 {
791 /* Write the VMCS revision dword to the VMXON region. */
792 *(uint32_t *)pvCpuPage = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
793 }
794
795 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
796 RTCCUINTREG fEFlags = ASMIntDisableFlags();
797
798 /* Enable the VMX bit in CR4 if necessary. */
799 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
800
801 /* Enter VMX root mode. */
802 int rc = VMXEnable(HCPhysCpuPage);
803 if (RT_FAILURE(rc))
804 {
805 if (!(uOldCr4 & X86_CR4_VMXE))
806 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
807
808 if (pVM)
809 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
810 }
811
812 /* Restore interrupts. */
813 ASMSetFlags(fEFlags);
814 return rc;
815}
816
817
818/**
819 * Exits VMX root mode operation on the current CPU.
820 *
821 * @returns VBox status code.
822 */
823static int hmR0VmxLeaveRootMode(void)
824{
825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
826
827 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
828 RTCCUINTREG fEFlags = ASMIntDisableFlags();
829
830 /* If we're for some reason not in VMX root mode, then don't leave it. */
831 RTCCUINTREG uHostCR4 = ASMGetCR4();
832
833 int rc;
834 if (uHostCR4 & X86_CR4_VMXE)
835 {
836 /* Exit VMX root mode and clear the VMX bit in CR4. */
837 VMXDisable();
838 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
839 rc = VINF_SUCCESS;
840 }
841 else
842 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
843
844 /* Restore interrupts. */
845 ASMSetFlags(fEFlags);
846 return rc;
847}
848
849
850/**
851 * Allocates and maps one physically contiguous page. The allocated page is
852 * zero'd out. (Used by various VT-x structures).
853 *
854 * @returns IPRT status code.
855 * @param pMemObj Pointer to the ring-0 memory object.
856 * @param ppVirt Where to store the virtual address of the
857 * allocation.
858 * @param pHCPhys Where to store the physical address of the
859 * allocation.
860 */
861static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
862{
863 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
864 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
865 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
866
867 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
868 if (RT_FAILURE(rc))
869 return rc;
870 *ppVirt = RTR0MemObjAddress(*pMemObj);
871 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
872 ASMMemZero32(*ppVirt, PAGE_SIZE);
873 return VINF_SUCCESS;
874}
875
876
877/**
878 * Frees and unmaps an allocated physical page.
879 *
880 * @param pMemObj Pointer to the ring-0 memory object.
881 * @param ppVirt Where to re-initialize the virtual address of
882 * allocation as 0.
883 * @param pHCPhys Where to re-initialize the physical address of the
884 * allocation as 0.
885 */
886static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
887{
888 AssertPtr(pMemObj);
889 AssertPtr(ppVirt);
890 AssertPtr(pHCPhys);
891 if (*pMemObj != NIL_RTR0MEMOBJ)
892 {
893 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
894 AssertRC(rc);
895 *pMemObj = NIL_RTR0MEMOBJ;
896 *ppVirt = 0;
897 *pHCPhys = 0;
898 }
899}
900
901
902/**
903 * Worker function to free VT-x related structures.
904 *
905 * @returns IPRT status code.
906 * @param pVM The cross context VM structure.
907 */
908static void hmR0VmxStructsFree(PVM pVM)
909{
910 for (VMCPUID i = 0; i < pVM->cCpus; i++)
911 {
912 PVMCPU pVCpu = &pVM->aCpus[i];
913 AssertPtr(pVCpu);
914
915 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
916 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
917
918 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
919 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
922 }
923
924 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
925#ifdef VBOX_WITH_CRASHDUMP_MAGIC
926 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
927#endif
928}
929
930
931/**
932 * Worker function to allocate VT-x related VM structures.
933 *
934 * @returns IPRT status code.
935 * @param pVM The cross context VM structure.
936 */
937static int hmR0VmxStructsAlloc(PVM pVM)
938{
939 /*
940 * Initialize members up-front so we can cleanup properly on allocation failure.
941 */
942#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
943 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
944 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
945 pVM->hm.s.vmx.HCPhys##a_Name = 0;
946
947#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
951
952#ifdef VBOX_WITH_CRASHDUMP_MAGIC
953 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
954#endif
955 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
956
957 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
958 for (VMCPUID i = 0; i < pVM->cCpus; i++)
959 {
960 PVMCPU pVCpu = &pVM->aCpus[i];
961 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
962 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
963 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
964 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
965 }
966#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
967#undef VMXLOCAL_INIT_VM_MEMOBJ
968
969 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
970 AssertReturnStmt(RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_SIZE) <= PAGE_SIZE,
971 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
972 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
973
974 /*
975 * Allocate all the VT-x structures.
976 */
977 int rc = VINF_SUCCESS;
978#ifdef VBOX_WITH_CRASHDUMP_MAGIC
979 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
980 if (RT_FAILURE(rc))
981 goto cleanup;
982 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
983 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
984#endif
985
986 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
987 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
988 {
989 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
990 &pVM->hm.s.vmx.HCPhysApicAccess);
991 if (RT_FAILURE(rc))
992 goto cleanup;
993 }
994
995 /*
996 * Initialize per-VCPU VT-x structures.
997 */
998 for (VMCPUID i = 0; i < pVM->cCpus; i++)
999 {
1000 PVMCPU pVCpu = &pVM->aCpus[i];
1001 AssertPtr(pVCpu);
1002
1003 /* Allocate the VM control structure (VMCS). */
1004 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1005 if (RT_FAILURE(rc))
1006 goto cleanup;
1007
1008 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1009 if ( PDMHasApic(pVM)
1010 && (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW))
1011 {
1012 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1013 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016 }
1017
1018 /*
1019 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1020 * transparent accesses of specific MSRs.
1021 *
1022 * If the condition for enabling MSR bitmaps changes here, don't forget to
1023 * update HMAreMsrBitmapsAvailable().
1024 */
1025 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1026 {
1027 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1028 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1029 if (RT_FAILURE(rc))
1030 goto cleanup;
1031 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1032 }
1033
1034 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1035 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038
1039 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043 }
1044
1045 return VINF_SUCCESS;
1046
1047cleanup:
1048 hmR0VmxStructsFree(pVM);
1049 return rc;
1050}
1051
1052
1053/**
1054 * Does global VT-x initialization (called during module initialization).
1055 *
1056 * @returns VBox status code.
1057 */
1058VMMR0DECL(int) VMXR0GlobalInit(void)
1059{
1060#ifdef HMVMX_USE_FUNCTION_TABLE
1061 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1062# ifdef VBOX_STRICT
1063 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1064 Assert(g_apfnVMExitHandlers[i]);
1065# endif
1066#endif
1067 return VINF_SUCCESS;
1068}
1069
1070
1071/**
1072 * Does global VT-x termination (called during module termination).
1073 */
1074VMMR0DECL(void) VMXR0GlobalTerm()
1075{
1076 /* Nothing to do currently. */
1077}
1078
1079
1080/**
1081 * Sets up and activates VT-x on the current CPU.
1082 *
1083 * @returns VBox status code.
1084 * @param pHostCpu Pointer to the global CPU info struct.
1085 * @param pVM The cross context VM structure. Can be
1086 * NULL after a host resume operation.
1087 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1088 * fEnabledByHost is @c true).
1089 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1090 * @a fEnabledByHost is @c true).
1091 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1092 * enable VT-x on the host.
1093 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1094 */
1095VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1096 void *pvMsrs)
1097{
1098 Assert(pHostCpu);
1099 Assert(pvMsrs);
1100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1101
1102 /* Enable VT-x if it's not already enabled by the host. */
1103 if (!fEnabledByHost)
1104 {
1105 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1106 if (RT_FAILURE(rc))
1107 return rc;
1108 }
1109
1110 /*
1111 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1112 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1113 * invalidated when flushing by VPID.
1114 */
1115 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1116 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1117 {
1118 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1119 pHostCpu->fFlushAsidBeforeUse = false;
1120 }
1121 else
1122 pHostCpu->fFlushAsidBeforeUse = true;
1123
1124 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1125 ++pHostCpu->cTlbFlushes;
1126
1127 return VINF_SUCCESS;
1128}
1129
1130
1131/**
1132 * Deactivates VT-x on the current CPU.
1133 *
1134 * @returns VBox status code.
1135 * @param pHostCpu Pointer to the global CPU info struct.
1136 * @param pvCpuPage Pointer to the VMXON region.
1137 * @param HCPhysCpuPage Physical address of the VMXON region.
1138 *
1139 * @remarks This function should never be called when SUPR0EnableVTx() or
1140 * similar was used to enable VT-x on the host.
1141 */
1142VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1143{
1144 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1145
1146 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1147 return hmR0VmxLeaveRootMode();
1148}
1149
1150
1151/**
1152 * Sets the permission bits for the specified MSR in the MSR bitmap.
1153 *
1154 * @param pVCpu The cross context virtual CPU structure.
1155 * @param uMsr The MSR value.
1156 * @param enmRead Whether reading this MSR causes a VM-exit.
1157 * @param enmWrite Whether writing this MSR causes a VM-exit.
1158 */
1159static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1160{
1161 int32_t iBit;
1162 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1163
1164 /*
1165 * Layout:
1166 * 0x000 - 0x3ff - Low MSR read bits
1167 * 0x400 - 0x7ff - High MSR read bits
1168 * 0x800 - 0xbff - Low MSR write bits
1169 * 0xc00 - 0xfff - High MSR write bits
1170 */
1171 if (uMsr <= 0x00001fff)
1172 iBit = uMsr;
1173 else if (uMsr - UINT32_C(0xc0000000) <= UINT32_C(0x00001fff))
1174 {
1175 iBit = uMsr - UINT32_C(0xc0000000);
1176 pbMsrBitmap += 0x400;
1177 }
1178 else
1179 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1180
1181 Assert(iBit <= 0x1fff);
1182 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1183 ASMBitSet(pbMsrBitmap, iBit);
1184 else
1185 ASMBitClear(pbMsrBitmap, iBit);
1186
1187 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1188 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1191}
1192
1193
1194#ifdef VBOX_STRICT
1195/**
1196 * Gets the permission bits for the specified MSR in the MSR bitmap.
1197 *
1198 * @returns VBox status code.
1199 * @retval VINF_SUCCESS if the specified MSR is found.
1200 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1201 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1202 *
1203 * @param pVCpu The cross context virtual CPU structure.
1204 * @param uMsr The MSR.
1205 * @param penmRead Where to store the read permissions.
1206 * @param penmWrite Where to store the write permissions.
1207 */
1208static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1209{
1210 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1211 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1212 int32_t iBit;
1213 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1214
1215 /* See hmR0VmxSetMsrPermission() for the layout. */
1216 if (uMsr <= 0x00001fff)
1217 iBit = uMsr;
1218 else if ( uMsr >= 0xc0000000
1219 && uMsr <= 0xc0001fff)
1220 {
1221 iBit = (uMsr - 0xc0000000);
1222 pbMsrBitmap += 0x400;
1223 }
1224 else
1225 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1226
1227 Assert(iBit <= 0x1fff);
1228 if (ASMBitTest(pbMsrBitmap, iBit))
1229 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1230 else
1231 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1232
1233 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1234 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1235 else
1236 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1237 return VINF_SUCCESS;
1238}
1239#endif /* VBOX_STRICT */
1240
1241
1242/**
1243 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1244 * area.
1245 *
1246 * @returns VBox status code.
1247 * @param pVCpu The cross context virtual CPU structure.
1248 * @param cMsrs The number of MSRs.
1249 */
1250static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1251{
1252 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1253 uint64_t const uVmxMiscMsr = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc;
1254 uint32_t const cMaxSupportedMsrs = VMX_MISC_MAX_MSRS(uVmxMiscMsr);
1255 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1256 {
1257 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1258 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1260 }
1261
1262 /* Update number of guest MSRs to load/store across the world-switch. */
1263 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1264 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1265
1266 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1267 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1268 AssertRCReturn(rc, rc);
1269
1270 /* Update the VCPU's copy of the MSR count. */
1271 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1272
1273 return VINF_SUCCESS;
1274}
1275
1276
1277/**
1278 * Adds a new (or updates the value of an existing) guest/host MSR
1279 * pair to be swapped during the world-switch as part of the
1280 * auto-load/store MSR area in the VMCS.
1281 *
1282 * @returns VBox status code.
1283 * @param pVCpu The cross context virtual CPU structure.
1284 * @param uMsr The MSR.
1285 * @param uGuestMsrValue Value of the guest MSR.
1286 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1287 * necessary.
1288 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1289 * its value was updated. Optional, can be NULL.
1290 */
1291static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1292 bool *pfAddedAndUpdated)
1293{
1294 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1295 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1296 uint32_t i;
1297 for (i = 0; i < cMsrs; i++)
1298 {
1299 if (pGuestMsr->u32Msr == uMsr)
1300 break;
1301 pGuestMsr++;
1302 }
1303
1304 bool fAdded = false;
1305 if (i == cMsrs)
1306 {
1307 ++cMsrs;
1308 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1309 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1310
1311 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1312 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1313 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1314
1315 fAdded = true;
1316 }
1317
1318 /* Update the MSR values in the auto-load/store MSR area. */
1319 pGuestMsr->u32Msr = uMsr;
1320 pGuestMsr->u64Value = uGuestMsrValue;
1321
1322 /* Create/update the MSR slot in the host MSR area. */
1323 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1324 pHostMsr += i;
1325 pHostMsr->u32Msr = uMsr;
1326
1327 /*
1328 * Update the host MSR only when requested by the caller AND when we're
1329 * adding it to the auto-load/store area. Otherwise, it would have been
1330 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1331 */
1332 bool fUpdatedMsrValue = false;
1333 if ( fAdded
1334 && fUpdateHostMsr)
1335 {
1336 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1337 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1338 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1339 fUpdatedMsrValue = true;
1340 }
1341
1342 if (pfAddedAndUpdated)
1343 *pfAddedAndUpdated = fUpdatedMsrValue;
1344 return VINF_SUCCESS;
1345}
1346
1347
1348/**
1349 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1350 * auto-load/store MSR area in the VMCS.
1351 *
1352 * @returns VBox status code.
1353 * @param pVCpu The cross context virtual CPU structure.
1354 * @param uMsr The MSR.
1355 */
1356static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1357{
1358 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1359 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1360 for (uint32_t i = 0; i < cMsrs; i++)
1361 {
1362 /* Find the MSR. */
1363 if (pGuestMsr->u32Msr == uMsr)
1364 {
1365 /* If it's the last MSR, simply reduce the count. */
1366 if (i == cMsrs - 1)
1367 {
1368 --cMsrs;
1369 break;
1370 }
1371
1372 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1373 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1374 pLastGuestMsr += cMsrs - 1;
1375 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1376 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1377
1378 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1379 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1380 pLastHostMsr += cMsrs - 1;
1381 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1382 pHostMsr->u64Value = pLastHostMsr->u64Value;
1383 --cMsrs;
1384 break;
1385 }
1386 pGuestMsr++;
1387 }
1388
1389 /* Update the VMCS if the count changed (meaning the MSR was found). */
1390 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1391 {
1392 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1393 AssertRCReturn(rc, rc);
1394
1395 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1396 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1397 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1398
1399 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1400 return VINF_SUCCESS;
1401 }
1402
1403 return VERR_NOT_FOUND;
1404}
1405
1406
1407/**
1408 * Checks if the specified guest MSR is part of the auto-load/store area in
1409 * the VMCS.
1410 *
1411 * @returns true if found, false otherwise.
1412 * @param pVCpu The cross context virtual CPU structure.
1413 * @param uMsr The MSR to find.
1414 */
1415static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1416{
1417 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1418 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1419
1420 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1421 {
1422 if (pGuestMsr->u32Msr == uMsr)
1423 return true;
1424 }
1425 return false;
1426}
1427
1428
1429/**
1430 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1431 *
1432 * @param pVCpu The cross context virtual CPU structure.
1433 *
1434 * @remarks No-long-jump zone!!!
1435 */
1436static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1437{
1438 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1439 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1440 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1441 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1442
1443 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1444 {
1445 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1446
1447 /*
1448 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1449 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1450 */
1451 if (pHostMsr->u32Msr == MSR_K6_EFER)
1452 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1453 else
1454 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1455 }
1456
1457 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1458}
1459
1460
1461/**
1462 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1463 * perform lazy restoration of the host MSRs while leaving VT-x.
1464 *
1465 * @param pVCpu The cross context virtual CPU structure.
1466 *
1467 * @remarks No-long-jump zone!!!
1468 */
1469static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1470{
1471 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1472
1473 /*
1474 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1475 */
1476 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1477 {
1478 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1479#if HC_ARCH_BITS == 64
1480 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1481 {
1482 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1483 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1484 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1485 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1486 }
1487#endif
1488 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1489 }
1490}
1491
1492
1493/**
1494 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1495 * lazily while leaving VT-x.
1496 *
1497 * @returns true if it does, false otherwise.
1498 * @param pVCpu The cross context virtual CPU structure.
1499 * @param uMsr The MSR to check.
1500 */
1501static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1502{
1503 NOREF(pVCpu);
1504#if HC_ARCH_BITS == 64
1505 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1506 {
1507 switch (uMsr)
1508 {
1509 case MSR_K8_LSTAR:
1510 case MSR_K6_STAR:
1511 case MSR_K8_SF_MASK:
1512 case MSR_K8_KERNEL_GS_BASE:
1513 return true;
1514 }
1515 }
1516#else
1517 RT_NOREF(pVCpu, uMsr);
1518#endif
1519 return false;
1520}
1521
1522
1523/**
1524 * Loads a set of guests MSRs to allow read/passthru to the guest.
1525 *
1526 * The name of this function is slightly confusing. This function does NOT
1527 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1528 * common prefix for functions dealing with "lazy restoration" of the shared
1529 * MSRs.
1530 *
1531 * @param pVCpu The cross context virtual CPU structure.
1532 *
1533 * @remarks No-long-jump zone!!!
1534 */
1535static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu)
1536{
1537 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1538 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1539
1540 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1541#if HC_ARCH_BITS == 64
1542 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1543 {
1544 /*
1545 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1546 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1547 * we can skip a few MSR writes.
1548 *
1549 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1550 * guest MSR values in the guest-CPU context might be different to what's currently
1551 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1552 * CPU, see @bugref{8728}.
1553 */
1554 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
1555 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1556 && pCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1557 && pCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1558 && pCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1559 && pCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1560 {
1561#ifdef VBOX_STRICT
1562 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pCtx->msrKERNELGSBASE);
1563 Assert(ASMRdMsr(MSR_K8_LSTAR) == pCtx->msrLSTAR);
1564 Assert(ASMRdMsr(MSR_K6_STAR) == pCtx->msrSTAR);
1565 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pCtx->msrSFMASK);
1566#endif
1567 }
1568 else
1569 {
1570 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE);
1571 ASMWrMsr(MSR_K8_LSTAR, pCtx->msrLSTAR);
1572 ASMWrMsr(MSR_K6_STAR, pCtx->msrSTAR);
1573 ASMWrMsr(MSR_K8_SF_MASK, pCtx->msrSFMASK);
1574 }
1575 }
1576#endif
1577 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1578}
1579
1580
1581/**
1582 * Performs lazy restoration of the set of host MSRs if they were previously
1583 * loaded with guest MSR values.
1584 *
1585 * @param pVCpu The cross context virtual CPU structure.
1586 *
1587 * @remarks No-long-jump zone!!!
1588 * @remarks The guest MSRs should have been saved back into the guest-CPU
1589 * context by hmR0VmxImportGuestState()!!!
1590 */
1591static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1592{
1593 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1594 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1595
1596 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1597 {
1598 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1599#if HC_ARCH_BITS == 64
1600 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1601 {
1602 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1603 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1604 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1605 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1606 }
1607#endif
1608 }
1609 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1610}
1611
1612
1613/**
1614 * Verifies that our cached values of the VMCS fields are all consistent with
1615 * what's actually present in the VMCS.
1616 *
1617 * @returns VBox status code.
1618 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1619 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1620 * VMCS content. HMCPU error-field is
1621 * updated, see VMX_VCI_XXX.
1622 * @param pVCpu The cross context virtual CPU structure.
1623 */
1624static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1625{
1626 uint32_t u32Val;
1627 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1628 AssertRCReturn(rc, rc);
1629 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1630 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1631 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1632 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1633
1634 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1635 AssertRCReturn(rc, rc);
1636 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1637 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1638 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1639 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1644 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1645 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1646 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1647
1648 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1651 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1652 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1653 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1662 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1663 }
1664
1665 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1666 AssertRCReturn(rc, rc);
1667 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1668 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1669 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1670 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1671
1672 uint64_t u64Val;
1673 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1674 AssertRCReturn(rc, rc);
1675 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1676 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1677 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1678 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1679
1680 return VINF_SUCCESS;
1681}
1682
1683
1684#ifdef VBOX_STRICT
1685/**
1686 * Verifies that our cached host EFER value has not changed
1687 * since we cached it.
1688 *
1689 * @param pVCpu The cross context virtual CPU structure.
1690 */
1691static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1692{
1693 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1694
1695 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_LOAD_EFER_MSR)
1696 {
1697 uint64_t u64Val;
1698 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1699 AssertRC(rc);
1700
1701 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1702 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1703 }
1704}
1705
1706
1707/**
1708 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1709 * VMCS are correct.
1710 *
1711 * @param pVCpu The cross context virtual CPU structure.
1712 */
1713static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1714{
1715 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1716
1717 /* Verify MSR counts in the VMCS are what we think it should be. */
1718 uint32_t cMsrs;
1719 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1720 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1721
1722 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1723 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1724
1725 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1726 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1727
1728 PCVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1729 PCVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1730 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1731 {
1732 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1733 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1734 pGuestMsr->u32Msr, cMsrs));
1735
1736 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1737 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1738 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1739
1740 /* Verify that the permissions are as expected in the MSR bitmap. */
1741 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
1742 {
1743 VMXMSREXITREAD enmRead;
1744 VMXMSREXITWRITE enmWrite;
1745 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1746 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1747 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1748 {
1749 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1750 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1751 }
1752 else
1753 {
1754 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1755 pGuestMsr->u32Msr, cMsrs));
1756 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1757 pGuestMsr->u32Msr, cMsrs));
1758 }
1759 }
1760 }
1761}
1762#endif /* VBOX_STRICT */
1763
1764
1765/**
1766 * Flushes the TLB using EPT.
1767 *
1768 * @returns VBox status code.
1769 * @param pVCpu The cross context virtual CPU structure of the calling
1770 * EMT. Can be NULL depending on @a enmTlbFlush.
1771 * @param enmTlbFlush Type of flush.
1772 *
1773 * @remarks Caller is responsible for making sure this function is called only
1774 * when NestedPaging is supported and providing @a enmTlbFlush that is
1775 * supported by the CPU.
1776 * @remarks Can be called with interrupts disabled.
1777 */
1778static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1779{
1780 uint64_t au64Descriptor[2];
1781 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1782 au64Descriptor[0] = 0;
1783 else
1784 {
1785 Assert(pVCpu);
1786 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1787 }
1788 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1789
1790 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1791 AssertMsg(rc == VINF_SUCCESS,
1792 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1793
1794 if ( RT_SUCCESS(rc)
1795 && pVCpu)
1796 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1797}
1798
1799
1800/**
1801 * Flushes the TLB using VPID.
1802 *
1803 * @returns VBox status code.
1804 * @param pVCpu The cross context virtual CPU structure of the calling
1805 * EMT. Can be NULL depending on @a enmTlbFlush.
1806 * @param enmTlbFlush Type of flush.
1807 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1808 * on @a enmTlbFlush).
1809 *
1810 * @remarks Can be called with interrupts disabled.
1811 */
1812static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1813{
1814 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1815
1816 uint64_t au64Descriptor[2];
1817 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1818 {
1819 au64Descriptor[0] = 0;
1820 au64Descriptor[1] = 0;
1821 }
1822 else
1823 {
1824 AssertPtr(pVCpu);
1825 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1826 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1827 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1828 au64Descriptor[1] = GCPtr;
1829 }
1830
1831 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1832 AssertMsg(rc == VINF_SUCCESS,
1833 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1834
1835 if ( RT_SUCCESS(rc)
1836 && pVCpu)
1837 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1838 NOREF(rc);
1839}
1840
1841
1842/**
1843 * Invalidates a guest page by guest virtual address. Only relevant for
1844 * EPT/VPID, otherwise there is nothing really to invalidate.
1845 *
1846 * @returns VBox status code.
1847 * @param pVCpu The cross context virtual CPU structure.
1848 * @param GCVirt Guest virtual address of the page to invalidate.
1849 */
1850VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1851{
1852 AssertPtr(pVCpu);
1853 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1854
1855 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1856 if (!fFlushPending)
1857 {
1858 /*
1859 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1860 * the EPT case. See @bugref{6043} and @bugref{6177}.
1861 *
1862 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1863 * as this function maybe called in a loop with individual addresses.
1864 */
1865 PVM pVM = pVCpu->CTX_SUFF(pVM);
1866 if (pVM->hm.s.vmx.fVpid)
1867 {
1868 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1869
1870#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1871 /*
1872 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1873 * where executing INVVPID outside 64-bit mode does not flush translations of
1874 * 64-bit linear addresses, see @bugref{6208#c72}.
1875 */
1876 if (RT_HI_U32(GCVirt))
1877 fVpidFlush = false;
1878#endif
1879
1880 if (fVpidFlush)
1881 {
1882 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1883 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1884 }
1885 else
1886 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1887 }
1888 else if (pVM->hm.s.fNestedPaging)
1889 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1890 }
1891
1892 return VINF_SUCCESS;
1893}
1894
1895
1896/**
1897 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1898 * case where neither EPT nor VPID is supported by the CPU.
1899 *
1900 * @param pVCpu The cross context virtual CPU structure.
1901 * @param pCpu Pointer to the global HM struct.
1902 *
1903 * @remarks Called with interrupts disabled.
1904 */
1905static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1906{
1907 AssertPtr(pVCpu);
1908 AssertPtr(pCpu);
1909
1910 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1911
1912 Assert(pCpu->idCpu != NIL_RTCPUID);
1913 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1914 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1915 pVCpu->hm.s.fForceTLBFlush = false;
1916 return;
1917}
1918
1919
1920/**
1921 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1922 *
1923 * @param pVCpu The cross context virtual CPU structure.
1924 * @param pCpu Pointer to the global HM CPU struct.
1925 *
1926 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1927 * nomenclature. The reason is, to avoid confusion in compare statements
1928 * since the host-CPU copies are named "ASID".
1929 *
1930 * @remarks Called with interrupts disabled.
1931 */
1932static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1933{
1934#ifdef VBOX_WITH_STATISTICS
1935 bool fTlbFlushed = false;
1936# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1937# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1938 if (!fTlbFlushed) \
1939 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1940 } while (0)
1941#else
1942# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1943# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1944#endif
1945
1946 AssertPtr(pCpu);
1947 AssertPtr(pVCpu);
1948 Assert(pCpu->idCpu != NIL_RTCPUID);
1949
1950 PVM pVM = pVCpu->CTX_SUFF(pVM);
1951 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1952 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1953 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1954
1955 /*
1956 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1957 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1958 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1959 * cannot reuse the current ASID anymore.
1960 */
1961 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1962 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1963 {
1964 ++pCpu->uCurrentAsid;
1965 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1966 {
1967 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1968 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1969 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1970 }
1971
1972 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1973 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1974 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1975
1976 /*
1977 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1978 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1984 }
1985 else if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH)) /* Check for explicit TLB flushes. */
1986 {
1987 /*
1988 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1989 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1990 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1991 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1992 * mappings, see @bugref{6568}.
1993 *
1994 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1995 */
1996 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1997 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1998 HMVMX_SET_TAGGED_TLB_FLUSHED();
1999 }
2000
2001 pVCpu->hm.s.fForceTLBFlush = false;
2002 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2003
2004 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2005 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2006 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2007 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2008 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2009 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2010 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2011 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2012 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2013
2014 /* Update VMCS with the VPID. */
2015 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2016 AssertRC(rc);
2017
2018#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2019}
2020
2021
2022/**
2023 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2024 *
2025 * @returns VBox status code.
2026 * @param pVCpu The cross context virtual CPU structure.
2027 * @param pCpu Pointer to the global HM CPU struct.
2028 *
2029 * @remarks Called with interrupts disabled.
2030 */
2031static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2032{
2033 AssertPtr(pVCpu);
2034 AssertPtr(pCpu);
2035 Assert(pCpu->idCpu != NIL_RTCPUID);
2036 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2037 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2038
2039 /*
2040 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2041 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2042 */
2043 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2044 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2045 {
2046 pVCpu->hm.s.fForceTLBFlush = true;
2047 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2048 }
2049
2050 /* Check for explicit TLB flushes. */
2051 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2052 {
2053 pVCpu->hm.s.fForceTLBFlush = true;
2054 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2055 }
2056
2057 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2058 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2059
2060 if (pVCpu->hm.s.fForceTLBFlush)
2061 {
2062 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2063 pVCpu->hm.s.fForceTLBFlush = false;
2064 }
2065}
2066
2067
2068/**
2069 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2070 *
2071 * @returns VBox status code.
2072 * @param pVCpu The cross context virtual CPU structure.
2073 * @param pCpu Pointer to the global HM CPU struct.
2074 *
2075 * @remarks Called with interrupts disabled.
2076 */
2077static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2078{
2079 AssertPtr(pVCpu);
2080 AssertPtr(pCpu);
2081 Assert(pCpu->idCpu != NIL_RTCPUID);
2082 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2083 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2084
2085 /*
2086 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2087 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2088 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2089 * cannot reuse the current ASID anymore.
2090 */
2091 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2092 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2093 {
2094 pVCpu->hm.s.fForceTLBFlush = true;
2095 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2096 }
2097
2098 /* Check for explicit TLB flushes. */
2099 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2100 {
2101 /*
2102 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2103 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2104 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2105 * include fExplicitFlush's too) - an obscure corner case.
2106 */
2107 pVCpu->hm.s.fForceTLBFlush = true;
2108 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2109 }
2110
2111 PVM pVM = pVCpu->CTX_SUFF(pVM);
2112 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2113 if (pVCpu->hm.s.fForceTLBFlush)
2114 {
2115 ++pCpu->uCurrentAsid;
2116 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2117 {
2118 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2119 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2120 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2121 }
2122
2123 pVCpu->hm.s.fForceTLBFlush = false;
2124 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2125 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2126 if (pCpu->fFlushAsidBeforeUse)
2127 {
2128 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2129 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2130 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2131 {
2132 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2133 pCpu->fFlushAsidBeforeUse = false;
2134 }
2135 else
2136 {
2137 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2138 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2139 }
2140 }
2141 }
2142
2143 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2144 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2145 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2146 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2147 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2148 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2149 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2150
2151 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2152 AssertRC(rc);
2153}
2154
2155
2156/**
2157 * Flushes the guest TLB entry based on CPU capabilities.
2158 *
2159 * @param pVCpu The cross context virtual CPU structure.
2160 * @param pCpu Pointer to the global HM CPU struct.
2161 */
2162DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2163{
2164#ifdef HMVMX_ALWAYS_FLUSH_TLB
2165 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2166#endif
2167 PVM pVM = pVCpu->CTX_SUFF(pVM);
2168 switch (pVM->hm.s.vmx.enmTlbFlushType)
2169 {
2170 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2171 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2172 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2173 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2174 default:
2175 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2176 break;
2177 }
2178 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2179}
2180
2181
2182/**
2183 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2184 * TLB entries from the host TLB before VM-entry.
2185 *
2186 * @returns VBox status code.
2187 * @param pVM The cross context VM structure.
2188 */
2189static int hmR0VmxSetupTaggedTlb(PVM pVM)
2190{
2191 /*
2192 * Determine optimal flush type for Nested Paging.
2193 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2194 * guest execution (see hmR3InitFinalizeR0()).
2195 */
2196 if (pVM->hm.s.fNestedPaging)
2197 {
2198 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2199 {
2200 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2201 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2202 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2203 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2204 else
2205 {
2206 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2207 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211
2212 /* Make sure the write-back cacheable memory type for EPT is supported. */
2213 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2214 {
2215 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219
2220 /* EPT requires a page-walk length of 4. */
2221 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2222 {
2223 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2224 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2225 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2226 }
2227 }
2228 else
2229 {
2230 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2231 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2232 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2233 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2234 }
2235 }
2236
2237 /*
2238 * Determine optimal flush type for VPID.
2239 */
2240 if (pVM->hm.s.vmx.fVpid)
2241 {
2242 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2243 {
2244 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2245 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2246 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2247 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2248 else
2249 {
2250 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2251 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2252 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2253 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2254 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2255 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2256 pVM->hm.s.vmx.fVpid = false;
2257 }
2258 }
2259 else
2260 {
2261 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2262 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2263 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2264 pVM->hm.s.vmx.fVpid = false;
2265 }
2266 }
2267
2268 /*
2269 * Setup the handler for flushing tagged-TLBs.
2270 */
2271 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2272 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2273 else if (pVM->hm.s.fNestedPaging)
2274 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2275 else if (pVM->hm.s.vmx.fVpid)
2276 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2277 else
2278 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2279 return VINF_SUCCESS;
2280}
2281
2282
2283/**
2284 * Sets up pin-based VM-execution controls in the VMCS.
2285 *
2286 * @returns VBox status code.
2287 * @param pVCpu The cross context virtual CPU structure.
2288 *
2289 * @remarks We don't really care about optimizing vmwrites here as it's done only
2290 * once per VM and hence we don't care about VMCS-field cache comparisons.
2291 */
2292static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2293{
2294 PVM pVM = pVCpu->CTX_SUFF(pVM);
2295 uint32_t fVal = pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0; /* Bits set here must always be set. */
2296 uint32_t const fZap = pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2297
2298 fVal |= VMX_PIN_CTLS_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2299 | VMX_PIN_CTLS_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2300
2301 if (pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_VIRT_NMI)
2302 fVal |= VMX_PIN_CTLS_VIRT_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2303
2304 /* Enable the VMX preemption timer. */
2305 if (pVM->hm.s.vmx.fUsePreemptTimer)
2306 {
2307 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_PREEMPT_TIMER);
2308 fVal |= VMX_PIN_CTLS_PREEMPT_TIMER;
2309 }
2310
2311#if 0
2312 /* Enable posted-interrupt processing. */
2313 if (pVM->hm.s.fPostedIntrs)
2314 {
2315 Assert(pVM->hm.s.vmx.Msrs.PinCtls.n.allowed1 & VMX_PIN_CTLS_POSTED_INT);
2316 Assert(pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_ACK_EXT_INT);
2317 fVal |= VMX_PIN_CTL_POSTED_INT;
2318 }
2319#endif
2320
2321 if ((fVal & fZap) != fVal)
2322 {
2323 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2324 pVM->hm.s.vmx.Msrs.PinCtls.n.disallowed0, fVal, fZap));
2325 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2326 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2327 }
2328
2329 /* Commit it to the VMCS and update our cache. */
2330 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2331 AssertRCReturn(rc, rc);
2332 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2333
2334 return VINF_SUCCESS;
2335}
2336
2337
2338/**
2339 * Sets up secondary processor-based VM-execution controls in the VMCS.
2340 *
2341 * @returns VBox status code.
2342 * @param pVCpu The cross context virtual CPU structure.
2343 *
2344 * @remarks We don't really care about optimizing vmwrites here as it's done only
2345 * once per VM and hence we don't care about VMCS-field cache comparisons.
2346 */
2347static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2348{
2349 PVM pVM = pVCpu->CTX_SUFF(pVM);
2350 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2351 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2352
2353 /* WBINVD causes a VM-exit. */
2354 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_WBINVD_EXIT)
2355 fVal |= VMX_PROC_CTLS2_WBINVD_EXIT;
2356
2357 /* Enable EPT (aka nested-paging). */
2358 if (pVM->hm.s.fNestedPaging)
2359 fVal |= VMX_PROC_CTLS2_EPT;
2360
2361 /*
2362 * Enable the INVPCID instruction if supported by the hardware and we expose
2363 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2364 */
2365 if ( (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_INVPCID)
2366 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2367 fVal |= VMX_PROC_CTLS2_INVPCID;
2368
2369 /* Enable VPID. */
2370 if (pVM->hm.s.vmx.fVpid)
2371 fVal |= VMX_PROC_CTLS2_VPID;
2372
2373 /* Enable Unrestricted guest execution. */
2374 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2375 fVal |= VMX_PROC_CTLS2_UNRESTRICTED_GUEST;
2376
2377#if 0
2378 if (pVM->hm.s.fVirtApicRegs)
2379 {
2380 /* Enable APIC-register virtualization. */
2381 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_APIC_REG_VIRT);
2382 fVal |= VMX_PROC_CTLS2_APIC_REG_VIRT;
2383
2384 /* Enable virtual-interrupt delivery. */
2385 Assert(pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_INTR_DELIVERY);
2386 fVal |= VMX_PROC_CTLS2_VIRT_INTR_DELIVERY;
2387 }
2388#endif
2389
2390 /* Virtualize-APIC accesses if supported by the CPU. The virtual-APIC page is where the TPR shadow resides. */
2391 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2392 * done dynamically. */
2393 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
2394 {
2395 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2396 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2397 fVal |= VMX_PROC_CTLS2_VIRT_APIC_ACCESS; /* Virtualize APIC accesses. */
2398 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2399 AssertRCReturn(rc, rc);
2400 }
2401
2402 /* Enable RDTSCP. */
2403 if (pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_RDTSCP)
2404 fVal |= VMX_PROC_CTLS2_RDTSCP;
2405
2406 /* Enable Pause-Loop exiting. */
2407 if ( pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1 & VMX_PROC_CTLS2_PAUSE_LOOP_EXIT
2408 && pVM->hm.s.vmx.cPleGapTicks
2409 && pVM->hm.s.vmx.cPleWindowTicks)
2410 {
2411 fVal |= VMX_PROC_CTLS2_PAUSE_LOOP_EXIT;
2412
2413 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2414 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2415 AssertRCReturn(rc, rc);
2416 }
2417
2418 if ((fVal & fZap) != fVal)
2419 {
2420 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2421 pVM->hm.s.vmx.Msrs.ProcCtls2.n.disallowed0, fVal, fZap));
2422 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2423 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2424 }
2425
2426 /* Commit it to the VMCS and update our cache. */
2427 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2428 AssertRCReturn(rc, rc);
2429 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2430
2431 return VINF_SUCCESS;
2432}
2433
2434
2435/**
2436 * Sets up processor-based VM-execution controls in the VMCS.
2437 *
2438 * @returns VBox status code.
2439 * @param pVCpu The cross context virtual CPU structure.
2440 *
2441 * @remarks We don't really care about optimizing vmwrites here as it's done only
2442 * once per VM and hence we don't care about VMCS-field cache comparisons.
2443 */
2444static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2445{
2446 PVM pVM = pVCpu->CTX_SUFF(pVM);
2447 uint32_t fVal = pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2448 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2449
2450 fVal |= VMX_PROC_CTLS_HLT_EXIT /* HLT causes a VM-exit. */
2451 | VMX_PROC_CTLS_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2452 | VMX_PROC_CTLS_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2453 | VMX_PROC_CTLS_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2454 | VMX_PROC_CTLS_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2455 | VMX_PROC_CTLS_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2456 | VMX_PROC_CTLS_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2457
2458 /* We toggle VMX_PROC_CTLS_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2459 if ( !(pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MOV_DR_EXIT)
2460 || (pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0 & VMX_PROC_CTLS_MOV_DR_EXIT))
2461 {
2462 LogRelFunc(("Unsupported VMX_PROC_CTLS_MOV_DR_EXIT combo!"));
2463 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2464 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2465 }
2466
2467 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2468 if (!pVM->hm.s.fNestedPaging)
2469 {
2470 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2471 fVal |= VMX_PROC_CTLS_INVLPG_EXIT
2472 | VMX_PROC_CTLS_CR3_LOAD_EXIT
2473 | VMX_PROC_CTLS_CR3_STORE_EXIT;
2474 }
2475
2476 /* Use TPR shadowing if supported by the CPU. */
2477 if ( PDMHasApic(pVM)
2478 && pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_TPR_SHADOW)
2479 {
2480 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2481 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2482 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2483 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2484 AssertRCReturn(rc, rc);
2485
2486 fVal |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2487 /* CR8 writes cause a VM-exit based on TPR threshold. */
2488 Assert(!(fVal & VMX_PROC_CTLS_CR8_STORE_EXIT));
2489 Assert(!(fVal & VMX_PROC_CTLS_CR8_LOAD_EXIT));
2490 }
2491 else
2492 {
2493 /*
2494 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2495 * Set this control only for 64-bit guests.
2496 */
2497 if (pVM->hm.s.fAllow64BitGuests)
2498 {
2499 fVal |= VMX_PROC_CTLS_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2500 | VMX_PROC_CTLS_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2501 }
2502 }
2503
2504 /* Use MSR-bitmaps if supported by the CPU. */
2505 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
2506 {
2507 fVal |= VMX_PROC_CTLS_USE_MSR_BITMAPS;
2508
2509 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2510 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2511 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2512 AssertRCReturn(rc, rc);
2513
2514 /*
2515 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2516 * automatically using dedicated fields in the VMCS.
2517 */
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2522 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2523#if HC_ARCH_BITS == 64
2524 /*
2525 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2526 */
2527 if (pVM->hm.s.fAllow64BitGuests)
2528 {
2529 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 }
2534#endif
2535 /*
2536 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2537 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2538 */
2539 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2540 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2541
2542 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2543 }
2544
2545 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2546 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2547 fVal |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
2548
2549 if ((fVal & fZap) != fVal)
2550 {
2551 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2552 pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0, fVal, fZap));
2553 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2554 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2555 }
2556
2557 /* Commit it to the VMCS and update our cache. */
2558 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2559 AssertRCReturn(rc, rc);
2560 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2561
2562 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2563 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
2564 return hmR0VmxSetupProcCtls2(pVCpu);
2565
2566 /* Sanity check, should not really happen. */
2567 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2568 {
2569 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2570 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2571 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2572 }
2573
2574 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2575 return VINF_SUCCESS;
2576}
2577
2578
2579/**
2580 * Sets up miscellaneous (everything other than Pin & Processor-based
2581 * VM-execution) control fields in the VMCS.
2582 *
2583 * @returns VBox status code.
2584 * @param pVCpu The cross context virtual CPU structure.
2585 */
2586static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2587{
2588 AssertPtr(pVCpu);
2589
2590 int rc = VERR_GENERAL_FAILURE;
2591
2592 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2593#if 0
2594 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2595 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2597
2598 /*
2599 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2600 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2601 * We thus use the exception bitmap to control it rather than use both.
2602 */
2603 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2604 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2605
2606 /* All IO & IOIO instructions cause VM-exits. */
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2608 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2609
2610 /* Initialize the MSR-bitmap area. */
2611 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2612 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2613 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2614 AssertRCReturn(rc, rc);
2615#endif
2616
2617 /* Setup MSR auto-load/store area. */
2618 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2619 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2621 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2622 AssertRCReturn(rc, rc);
2623
2624 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2625 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2626 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2627 AssertRCReturn(rc, rc);
2628
2629 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2630 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2631 AssertRCReturn(rc, rc);
2632
2633 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2634#if 0
2635 /* Setup debug controls */
2636 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2637 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2638 AssertRCReturn(rc, rc);
2639#endif
2640
2641 return rc;
2642}
2643
2644
2645/**
2646 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2647 *
2648 * We shall setup those exception intercepts that don't change during the
2649 * lifetime of the VM here. The rest are done dynamically while loading the
2650 * guest state.
2651 *
2652 * @returns VBox status code.
2653 * @param pVCpu The cross context virtual CPU structure.
2654 */
2655static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2656{
2657 AssertPtr(pVCpu);
2658
2659 uint32_t uXcptBitmap;
2660
2661 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2662 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2663
2664 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2665 and writes, and because recursive #DBs can cause the CPU hang, we must always
2666 intercept #DB. */
2667 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2668
2669 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2670 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2671 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2672
2673 /* Commit it to the VMCS. */
2674 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2675 AssertRCReturn(rc, rc);
2676
2677 /* Update our cache of the exception bitmap. */
2678 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2679 return VINF_SUCCESS;
2680}
2681
2682
2683/**
2684 * Does per-VM VT-x initialization.
2685 *
2686 * @returns VBox status code.
2687 * @param pVM The cross context VM structure.
2688 */
2689VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2690{
2691 LogFlowFunc(("pVM=%p\n", pVM));
2692
2693 int rc = hmR0VmxStructsAlloc(pVM);
2694 if (RT_FAILURE(rc))
2695 {
2696 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2697 return rc;
2698 }
2699
2700 return VINF_SUCCESS;
2701}
2702
2703
2704/**
2705 * Does per-VM VT-x termination.
2706 *
2707 * @returns VBox status code.
2708 * @param pVM The cross context VM structure.
2709 */
2710VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2711{
2712 LogFlowFunc(("pVM=%p\n", pVM));
2713
2714#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2715 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2716 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2717#endif
2718 hmR0VmxStructsFree(pVM);
2719 return VINF_SUCCESS;
2720}
2721
2722
2723/**
2724 * Sets up the VM for execution under VT-x.
2725 * This function is only called once per-VM during initialization.
2726 *
2727 * @returns VBox status code.
2728 * @param pVM The cross context VM structure.
2729 */
2730VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2731{
2732 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2733 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2734
2735 LogFlowFunc(("pVM=%p\n", pVM));
2736
2737 /*
2738 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2739 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2740 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2741 */
2742 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2743 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2744 || !pVM->hm.s.vmx.pRealModeTSS))
2745 {
2746 LogRelFunc(("Invalid real-on-v86 state.\n"));
2747 return VERR_INTERNAL_ERROR;
2748 }
2749
2750 /* Initialize these always, see hmR3InitFinalizeR0().*/
2751 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2752 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2753
2754 /* Setup the tagged-TLB flush handlers. */
2755 int rc = hmR0VmxSetupTaggedTlb(pVM);
2756 if (RT_FAILURE(rc))
2757 {
2758 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2759 return rc;
2760 }
2761
2762 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2763 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2764#if HC_ARCH_BITS == 64
2765 if ( (pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1 & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
2766 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_LOAD_EFER_MSR)
2767 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_EFER_MSR))
2768 {
2769 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2770 }
2771#endif
2772
2773 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2774 RTCCUINTREG uHostCR4 = ASMGetCR4();
2775 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2776 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2777
2778 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2779 {
2780 PVMCPU pVCpu = &pVM->aCpus[i];
2781 AssertPtr(pVCpu);
2782 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2783
2784 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2785 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2786
2787 /* Set revision dword at the beginning of the VMCS structure. */
2788 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_ID);
2789
2790 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2791 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2792 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2793 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2794
2795 /* Load this VMCS as the current VMCS. */
2796 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800 rc = hmR0VmxSetupPinCtls(pVCpu);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2802 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2803
2804 rc = hmR0VmxSetupProcCtls(pVCpu);
2805 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2806 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2807
2808 rc = hmR0VmxSetupMiscCtls(pVCpu);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2810 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2811
2812 rc = hmR0VmxInitXcptBitmap(pVCpu);
2813 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2815
2816#if HC_ARCH_BITS == 32
2817 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2818 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2819 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2820#endif
2821
2822 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2823 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2824 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2825 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2826
2827 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2828
2829 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2830 }
2831
2832 return VINF_SUCCESS;
2833}
2834
2835
2836/**
2837 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2838 * the VMCS.
2839 *
2840 * @returns VBox status code.
2841 */
2842static int hmR0VmxExportHostControlRegs(void)
2843{
2844 RTCCUINTREG uReg = ASMGetCR0();
2845 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2846 AssertRCReturn(rc, rc);
2847
2848 uReg = ASMGetCR3();
2849 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2850 AssertRCReturn(rc, rc);
2851
2852 uReg = ASMGetCR4();
2853 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2854 AssertRCReturn(rc, rc);
2855 return rc;
2856}
2857
2858
2859/**
2860 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2861 * the host-state area in the VMCS.
2862 *
2863 * @returns VBox status code.
2864 * @param pVCpu The cross context virtual CPU structure.
2865 */
2866static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2867{
2868#if HC_ARCH_BITS == 64
2869/**
2870 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2871 * requirements. See hmR0VmxExportHostSegmentRegs().
2872 */
2873# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2874 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2875 { \
2876 bool fValidSelector = true; \
2877 if ((selValue) & X86_SEL_LDT) \
2878 { \
2879 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2880 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2881 } \
2882 if (fValidSelector) \
2883 { \
2884 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2885 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2886 } \
2887 (selValue) = 0; \
2888 }
2889
2890 /*
2891 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2892 * should -not- save the messed up state without restoring the original host-state,
2893 * see @bugref{7240}.
2894 *
2895 * This apparently can happen (most likely the FPU changes), deal with it rather than
2896 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2897 */
2898 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2899 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2900 {
2901 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2902 pVCpu->idCpu));
2903 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2904 }
2905 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2906#else
2907 RT_NOREF(pVCpu);
2908#endif
2909
2910 /*
2911 * Host DS, ES, FS and GS segment registers.
2912 */
2913#if HC_ARCH_BITS == 64
2914 RTSEL uSelDS = ASMGetDS();
2915 RTSEL uSelES = ASMGetES();
2916 RTSEL uSelFS = ASMGetFS();
2917 RTSEL uSelGS = ASMGetGS();
2918#else
2919 RTSEL uSelDS = 0;
2920 RTSEL uSelES = 0;
2921 RTSEL uSelFS = 0;
2922 RTSEL uSelGS = 0;
2923#endif
2924
2925 /*
2926 * Host CS and SS segment registers.
2927 */
2928 RTSEL uSelCS = ASMGetCS();
2929 RTSEL uSelSS = ASMGetSS();
2930
2931 /*
2932 * Host TR segment register.
2933 */
2934 RTSEL uSelTR = ASMGetTR();
2935
2936#if HC_ARCH_BITS == 64
2937 /*
2938 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2939 * gain VM-entry and restore them before we get preempted.
2940 *
2941 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2942 */
2943 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2944 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2945 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2946 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2947# undef VMXLOCAL_ADJUST_HOST_SEG
2948#endif
2949
2950 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2951 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2952 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2953 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2954 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2955 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2956 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2957 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2958 Assert(uSelCS);
2959 Assert(uSelTR);
2960
2961 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2962#if 0
2963 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE))
2964 Assert(uSelSS != 0);
2965#endif
2966
2967 /* Write these host selector fields into the host-state area in the VMCS. */
2968 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2970#if HC_ARCH_BITS == 64
2971 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2972 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2973 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2974 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2975#else
2976 NOREF(uSelDS);
2977 NOREF(uSelES);
2978 NOREF(uSelFS);
2979 NOREF(uSelGS);
2980#endif
2981 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2982 AssertRCReturn(rc, rc);
2983
2984 /*
2985 * Host GDTR and IDTR.
2986 */
2987 RTGDTR Gdtr;
2988 RTIDTR Idtr;
2989 RT_ZERO(Gdtr);
2990 RT_ZERO(Idtr);
2991 ASMGetGDTR(&Gdtr);
2992 ASMGetIDTR(&Idtr);
2993 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2994 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2995 AssertRCReturn(rc, rc);
2996
2997#if HC_ARCH_BITS == 64
2998 /*
2999 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3000 * them to the maximum limit (0xffff) on every VM-exit.
3001 */
3002 if (Gdtr.cbGdt != 0xffff)
3003 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3004
3005 /*
3006 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3007 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3008 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3009 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3010 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3011 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3012 * at 0xffff on hosts where we are sure it won't cause trouble.
3013 */
3014# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3015 if (Idtr.cbIdt < 0x0fff)
3016# else
3017 if (Idtr.cbIdt != 0xffff)
3018# endif
3019 {
3020 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3021 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3022 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3023 }
3024#endif
3025
3026 /*
3027 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3028 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3029 * RPL should be too in most cases.
3030 */
3031 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3032 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3033
3034 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3035#if HC_ARCH_BITS == 64
3036 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3037
3038 /*
3039 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3040 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3041 * restoration if the host has something else. Task switching is not supported in 64-bit
3042 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3043 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3044 *
3045 * [1] See Intel spec. 3.5 "System Descriptor Types".
3046 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3047 */
3048 PVM pVM = pVCpu->CTX_SUFF(pVM);
3049 Assert(pDesc->System.u4Type == 11);
3050 if ( pDesc->System.u16LimitLow != 0x67
3051 || pDesc->System.u4LimitHigh)
3052 {
3053 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3054 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3055 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3056 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3057 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3058 }
3059
3060 /*
3061 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3062 */
3063 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3064 {
3065 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3066 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3067 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3068 {
3069 /* The GDT is read-only but the writable GDT is available. */
3070 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3071 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3072 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3073 AssertRCReturn(rc, rc);
3074 }
3075 }
3076#else
3077 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3078#endif
3079 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3080 AssertRCReturn(rc, rc);
3081
3082 /*
3083 * Host FS base and GS base.
3084 */
3085#if HC_ARCH_BITS == 64
3086 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3087 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3088 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3089 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3090 AssertRCReturn(rc, rc);
3091
3092 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3093 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3094 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3095 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3096 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3097#endif
3098 return VINF_SUCCESS;
3099}
3100
3101
3102/**
3103 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3104 * host-state area of the VMCS.
3105 *
3106 * Theses MSRs will be automatically restored on the host after every successful
3107 * VM-exit.
3108 *
3109 * @returns VBox status code.
3110 * @param pVCpu The cross context virtual CPU structure.
3111 *
3112 * @remarks No-long-jump zone!!!
3113 */
3114static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3115{
3116 AssertPtr(pVCpu);
3117 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3118
3119 /*
3120 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3121 * rather than swapping them on every VM-entry.
3122 */
3123 hmR0VmxLazySaveHostMsrs(pVCpu);
3124
3125 /*
3126 * Host Sysenter MSRs.
3127 */
3128 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3129#if HC_ARCH_BITS == 32
3130 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3131 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3132#else
3133 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3134 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3135#endif
3136 AssertRCReturn(rc, rc);
3137
3138 /*
3139 * Host EFER MSR.
3140 *
3141 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3142 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3143 */
3144 PVM pVM = pVCpu->CTX_SUFF(pVM);
3145 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3146 {
3147 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3148 AssertRCReturn(rc, rc);
3149 }
3150
3151 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3152
3153 return VINF_SUCCESS;
3154}
3155
3156
3157/**
3158 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3159 *
3160 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3161 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3162 * hmR0VMxExportGuestEntryCtls().
3163 *
3164 * @returns true if we need to load guest EFER, false otherwise.
3165 * @param pVCpu The cross context virtual CPU structure.
3166 *
3167 * @remarks Requires EFER, CR4.
3168 * @remarks No-long-jump zone!!!
3169 */
3170static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu)
3171{
3172#ifdef HMVMX_ALWAYS_SWAP_EFER
3173 return true;
3174#endif
3175
3176 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3177#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3178 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3179 if (CPUMIsGuestInLongModeEx(pCtx))
3180 return false;
3181#endif
3182
3183 PVM pVM = pVCpu->CTX_SUFF(pVM);
3184 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3185 uint64_t const u64GuestEfer = pCtx->msrEFER;
3186
3187 /*
3188 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3189 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3190 */
3191 if ( CPUMIsGuestInLongModeEx(pCtx)
3192 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3193 {
3194 return true;
3195 }
3196
3197 /*
3198 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3199 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3200 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3201 */
3202 if ( (pCtx->cr4 & X86_CR4_PAE)
3203 && (pCtx->cr0 & X86_CR0_PG)
3204 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3205 {
3206 /* Assert that host is NX capable. */
3207 Assert(pVCpu->CTX_SUFF(pVM)->cpum.ro.HostFeatures.fNoExecute);
3208 return true;
3209 }
3210
3211 return false;
3212}
3213
3214
3215/**
3216 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3217 *
3218 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3219 * see Intel spec. 24.8.1 "VM-entry controls".
3220 *
3221 * @returns VBox status code.
3222 * @param pVCpu The cross context virtual CPU structure.
3223 *
3224 * @remarks Requires EFER.
3225 * @remarks No-long-jump zone!!!
3226 */
3227static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu)
3228{
3229 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3230 {
3231 PVM pVM = pVCpu->CTX_SUFF(pVM);
3232 uint32_t fVal = pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3233 uint32_t const fZap = pVM->hm.s.vmx.Msrs.EntryCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3234
3235 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3236 fVal |= VMX_ENTRY_CTLS_LOAD_DEBUG;
3237
3238 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3239 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
3240 {
3241 fVal |= VMX_ENTRY_CTLS_IA32E_MODE_GUEST;
3242 Log4Func(("VMX_ENTRY_CTLS_IA32E_MODE_GUEST\n"));
3243 }
3244 else
3245 Assert(!(fVal & VMX_ENTRY_CTLS_IA32E_MODE_GUEST));
3246
3247 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3248 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3249 && hmR0VmxShouldSwapEferMsr(pVCpu))
3250 {
3251 fVal |= VMX_ENTRY_CTLS_LOAD_EFER_MSR;
3252 Log4Func(("VMX_ENTRY_CTLS_LOAD_EFER_MSR\n"));
3253 }
3254
3255 /*
3256 * The following should -not- be set (since we're not in SMM mode):
3257 * - VMX_ENTRY_CTLS_ENTRY_TO_SMM
3258 * - VMX_ENTRY_CTLS_DEACTIVATE_DUAL_MON
3259 */
3260
3261 /** @todo VMX_ENTRY_CTLS_LOAD_PERF_MSR,
3262 * VMX_ENTRY_CTLS_LOAD_PAT_MSR. */
3263
3264 if ((fVal & fZap) != fVal)
3265 {
3266 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3267 pVM->hm.s.vmx.Msrs.EntryCtls.n.disallowed0, fVal, fZap));
3268 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3269 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3270 }
3271
3272 /* Commit it to the VMCS and update our cache. */
3273 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3274 {
3275 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3276 AssertRCReturn(rc, rc);
3277 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3278 }
3279
3280 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3281 }
3282 return VINF_SUCCESS;
3283}
3284
3285
3286/**
3287 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3288 *
3289 * @returns VBox status code.
3290 * @param pVCpu The cross context virtual CPU structure.
3291 *
3292 * @remarks Requires EFER.
3293 */
3294static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu)
3295{
3296 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3297 {
3298 PVM pVM = pVCpu->CTX_SUFF(pVM);
3299 uint32_t fVal = pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
3300 uint32_t const fZap = pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3301
3302 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3303 fVal |= VMX_EXIT_CTLS_SAVE_DEBUG;
3304
3305 /*
3306 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3307 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3308 * hmR0VmxExportHostMsrs().
3309 */
3310#if HC_ARCH_BITS == 64
3311 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3312 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3313#else
3314 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3315 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3316 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3317 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3318 {
3319 /* The switcher returns to long mode, EFER is managed by the switcher. */
3320 fVal |= VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE;
3321 Log4Func(("VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE\n"));
3322 }
3323 else
3324 Assert(!(fVal & VMX_EXIT_CTLS_HOST_ADDR_SPACE_SIZE));
3325#endif
3326
3327 /* If the newer VMCS fields for managing EFER exists, use it. */
3328 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3329 && hmR0VmxShouldSwapEferMsr(pVCpu))
3330 {
3331 fVal |= VMX_EXIT_CTLS_SAVE_EFER_MSR
3332 | VMX_EXIT_CTLS_LOAD_EFER_MSR;
3333 Log4Func(("VMX_EXIT_CTLS_SAVE_EFER_MSR and VMX_EXIT_CTLS_LOAD_EFER_MSR\n"));
3334 }
3335
3336 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3337 Assert(!(fVal & VMX_EXIT_CTLS_ACK_EXT_INT));
3338
3339 /** @todo VMX_EXIT_CTLS_LOAD_PERF_MSR,
3340 * VMX_EXIT_CTLS_SAVE_PAT_MSR,
3341 * VMX_EXIT_CTLS_LOAD_PAT_MSR. */
3342
3343 /* Enable saving of the VMX preemption timer value on VM-exit. */
3344 if ( pVM->hm.s.vmx.fUsePreemptTimer
3345 && (pVM->hm.s.vmx.Msrs.ExitCtls.n.allowed1 & VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER))
3346 fVal |= VMX_EXIT_CTLS_SAVE_VMX_PREEMPT_TIMER;
3347
3348 if ((fVal & fZap) != fVal)
3349 {
3350 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3351 pVM->hm.s.vmx.Msrs.ExitCtls.n.disallowed0, fVal, fZap));
3352 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3353 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3354 }
3355
3356 /* Commit it to the VMCS and update our cache. */
3357 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3358 {
3359 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3360 AssertRCReturn(rc, rc);
3361 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3362 }
3363
3364 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3365 }
3366 return VINF_SUCCESS;
3367}
3368
3369
3370/**
3371 * Sets the TPR threshold in the VMCS.
3372 *
3373 * @returns VBox status code.
3374 * @param pVCpu The cross context virtual CPU structure.
3375 * @param u32TprThreshold The TPR threshold (task-priority class only).
3376 */
3377DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3378{
3379 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3380 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3381 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3382}
3383
3384
3385/**
3386 * Exports the guest APIC TPR state into the VMCS.
3387 *
3388 * @returns VBox status code.
3389 * @param pVCpu The cross context virtual CPU structure.
3390 *
3391 * @remarks No-long-jump zone!!!
3392 */
3393static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3394{
3395 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3396 {
3397 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3398
3399 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3400 && APICIsEnabled(pVCpu))
3401 {
3402 /*
3403 * Setup TPR shadowing.
3404 */
3405 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
3406 {
3407 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3408
3409 bool fPendingIntr = false;
3410 uint8_t u8Tpr = 0;
3411 uint8_t u8PendingIntr = 0;
3412 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3413 AssertRCReturn(rc, rc);
3414
3415 /*
3416 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3417 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3418 * priority of the pending interrupt so we can deliver the interrupt. If there
3419 * are no interrupts pending, set threshold to 0 to not cause any
3420 * TPR-below-threshold VM-exits.
3421 */
3422 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3423 uint32_t u32TprThreshold = 0;
3424 if (fPendingIntr)
3425 {
3426 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3427 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3428 const uint8_t u8TprPriority = u8Tpr >> 4;
3429 if (u8PendingPriority <= u8TprPriority)
3430 u32TprThreshold = u8PendingPriority;
3431 }
3432
3433 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3434 AssertRCReturn(rc, rc);
3435 }
3436 }
3437 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3438 }
3439 return VINF_SUCCESS;
3440}
3441
3442
3443/**
3444 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3445 *
3446 * @returns Guest's interruptibility-state.
3447 * @param pVCpu The cross context virtual CPU structure.
3448 *
3449 * @remarks No-long-jump zone!!!
3450 */
3451static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu)
3452{
3453 /*
3454 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3455 */
3456 uint32_t fIntrState = 0;
3457 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3458 {
3459 /* If inhibition is active, RIP & RFLAGS should've been accessed
3460 (i.e. read previously from the VMCS or from ring-3). */
3461 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3462#ifdef VBOX_STRICT
3463 uint64_t const fExtrn = ASMAtomicUoReadU64(&pCtx->fExtrn);
3464 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3465#endif
3466 if (pCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3467 {
3468 if (pCtx->eflags.Bits.u1IF)
3469 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
3470 else
3471 fIntrState = VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS;
3472 }
3473 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3474 {
3475 /*
3476 * We can clear the inhibit force flag as even if we go back to the recompiler
3477 * without executing guest code in VT-x, the flag's condition to be cleared is
3478 * met and thus the cleared state is correct.
3479 */
3480 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3481 }
3482 }
3483
3484 /*
3485 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3486 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3487 * setting this would block host-NMIs and IRET will not clear the blocking.
3488 *
3489 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3490 */
3491 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3492 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
3493 {
3494 fIntrState |= VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI;
3495 }
3496
3497 return fIntrState;
3498}
3499
3500
3501/**
3502 * Exports the guest's interruptibility-state into the guest-state area in the
3503 * VMCS.
3504 *
3505 * @returns VBox status code.
3506 * @param pVCpu The cross context virtual CPU structure.
3507 * @param fIntrState The interruptibility-state to set.
3508 */
3509static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3510{
3511 NOREF(pVCpu);
3512 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3513 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3514 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INT_STATE, fIntrState);
3515}
3516
3517
3518/**
3519 * Exports the exception intercepts required for guest execution in the VMCS.
3520 *
3521 * @returns VBox status code.
3522 * @param pVCpu The cross context virtual CPU structure.
3523 *
3524 * @remarks No-long-jump zone!!!
3525 */
3526static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3527{
3528 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3529 {
3530 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3531
3532 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3533 if (pVCpu->hm.s.fGIMTrapXcptUD)
3534 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3535#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3536 else
3537 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3538#endif
3539
3540 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3541 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3542
3543 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3544 {
3545 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3546 AssertRCReturn(rc, rc);
3547 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3548 }
3549
3550 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3551 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3552 }
3553 return VINF_SUCCESS;
3554}
3555
3556
3557/**
3558 * Exports the guest's RIP into the guest-state area in the VMCS.
3559 *
3560 * @returns VBox status code.
3561 * @param pVCpu The cross context virtual CPU structure.
3562 *
3563 * @remarks No-long-jump zone!!!
3564 */
3565static int hmR0VmxExportGuestRip(PVMCPU pVCpu)
3566{
3567 int rc = VINF_SUCCESS;
3568 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3569 {
3570 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3571
3572 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pVCpu->cpum.GstCtx.rip);
3573 AssertRCReturn(rc, rc);
3574
3575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3576 Log4Func(("RIP=%#RX64\n", pVCpu->cpum.GstCtx.rip));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Exports the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu The cross context virtual CPU structure.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxExportGuestRsp(PVMCPU pVCpu)
3591{
3592 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3593 {
3594 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3595
3596 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pVCpu->cpum.GstCtx.rsp);
3597 AssertRCReturn(rc, rc);
3598
3599 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3600 }
3601 return VINF_SUCCESS;
3602}
3603
3604
3605/**
3606 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3607 *
3608 * @returns VBox status code.
3609 * @param pVCpu The cross context virtual CPU structure.
3610 *
3611 * @remarks No-long-jump zone!!!
3612 */
3613static int hmR0VmxExportGuestRflags(PVMCPU pVCpu)
3614{
3615 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3616 {
3617 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3618
3619 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3620 Let us assert it as such and use 32-bit VMWRITE. */
3621 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.rflags.u64));
3622 X86EFLAGS fEFlags = pVCpu->cpum.GstCtx.eflags;
3623 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3624 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3625
3626 /*
3627 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3628 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3629 * can run the real-mode guest code under Virtual 8086 mode.
3630 */
3631 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3632 {
3633 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3634 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3635 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3636 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3637 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3638 }
3639
3640 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3641 AssertRCReturn(rc, rc);
3642
3643 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3644 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3645 }
3646 return VINF_SUCCESS;
3647}
3648
3649
3650/**
3651 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3652 *
3653 * The guest FPU state is always pre-loaded hence we don't need to bother about
3654 * sharing FPU related CR0 bits between the guest and host.
3655 *
3656 * @returns VBox status code.
3657 * @param pVCpu The cross context virtual CPU structure.
3658 *
3659 * @remarks No-long-jump zone!!!
3660 */
3661static int hmR0VmxExportGuestCR0(PVMCPU pVCpu)
3662{
3663 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3664 {
3665 PVM pVM = pVCpu->CTX_SUFF(pVM);
3666 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3667 Assert(!RT_HI_U32(pVCpu->cpum.GstCtx.cr0));
3668
3669 uint32_t const u32ShadowCr0 = pVCpu->cpum.GstCtx.cr0;
3670 uint32_t u32GuestCr0 = pVCpu->cpum.GstCtx.cr0;
3671
3672 /*
3673 * Setup VT-x's view of the guest CR0.
3674 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3675 */
3676 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3677 if (pVM->hm.s.fNestedPaging)
3678 {
3679 if (CPUMIsGuestPagingEnabled(pVCpu))
3680 {
3681 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3682 uProcCtls &= ~( VMX_PROC_CTLS_CR3_LOAD_EXIT
3683 | VMX_PROC_CTLS_CR3_STORE_EXIT);
3684 }
3685 else
3686 {
3687 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3688 uProcCtls |= VMX_PROC_CTLS_CR3_LOAD_EXIT
3689 | VMX_PROC_CTLS_CR3_STORE_EXIT;
3690 }
3691
3692 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3693 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3694 uProcCtls &= ~VMX_PROC_CTLS_CR3_STORE_EXIT;
3695 }
3696 else
3697 {
3698 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3699 u32GuestCr0 |= X86_CR0_WP;
3700 }
3701
3702 /*
3703 * Guest FPU bits.
3704 *
3705 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3706 * using CR0.TS.
3707 *
3708 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3709 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3710 */
3711 u32GuestCr0 |= X86_CR0_NE;
3712
3713 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3714 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3715
3716 /*
3717 * Update exception intercepts.
3718 */
3719 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721 {
3722 Assert(PDMVmmDevHeapIsEnabled(pVM));
3723 Assert(pVM->hm.s.vmx.pRealModeTSS);
3724 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3725 }
3726 else
3727 {
3728 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3729 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3730 if (fInterceptMF)
3731 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3732 }
3733
3734 /* Additional intercepts for debugging, define these yourself explicitly. */
3735#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3736 uXcptBitmap |= 0
3737 | RT_BIT(X86_XCPT_BP)
3738 | RT_BIT(X86_XCPT_DE)
3739 | RT_BIT(X86_XCPT_NM)
3740 | RT_BIT(X86_XCPT_TS)
3741 | RT_BIT(X86_XCPT_UD)
3742 | RT_BIT(X86_XCPT_NP)
3743 | RT_BIT(X86_XCPT_SS)
3744 | RT_BIT(X86_XCPT_GP)
3745 | RT_BIT(X86_XCPT_PF)
3746 | RT_BIT(X86_XCPT_MF)
3747 ;
3748#elif defined(HMVMX_ALWAYS_TRAP_PF)
3749 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3750#endif
3751 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3752
3753 /*
3754 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3755 */
3756 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3757 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3759 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3760 else
3761 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3762
3763 u32GuestCr0 |= fSetCr0;
3764 u32GuestCr0 &= fZapCr0;
3765 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3766
3767 /*
3768 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3769 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3770 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3771 */
3772 uint32_t u32Cr0Mask = X86_CR0_PE
3773 | X86_CR0_NE
3774 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3775 | X86_CR0_PG
3776 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3777 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3778 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3779
3780 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3781 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3782 * and @bugref{6944}. */
3783#if 0
3784 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3785 u32Cr0Mask &= ~X86_CR0_PE;
3786#endif
3787 /*
3788 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3789 */
3790 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3791 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3792 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3793 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3794 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3795 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3796 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3797 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3798 AssertRCReturn(rc, rc);
3799
3800 /* Update our caches. */
3801 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3802 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3803 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3804
3805 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3806
3807 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3808 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3809 }
3810
3811 return VINF_SUCCESS;
3812}
3813
3814
3815/**
3816 * Exports the guest control registers (CR3, CR4) into the guest-state area
3817 * in the VMCS.
3818 *
3819 * @returns VBox strict status code.
3820 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3821 * without unrestricted guest access and the VMMDev is not presently
3822 * mapped (e.g. EFI32).
3823 *
3824 * @param pVCpu The cross context virtual CPU structure.
3825 *
3826 * @remarks No-long-jump zone!!!
3827 */
3828static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu)
3829{
3830 int rc = VINF_SUCCESS;
3831 PVM pVM = pVCpu->CTX_SUFF(pVM);
3832
3833 /*
3834 * Guest CR2.
3835 * It's always loaded in the assembler code. Nothing to do here.
3836 */
3837
3838 /*
3839 * Guest CR3.
3840 */
3841 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3842 {
3843 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3844
3845 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3846 if (pVM->hm.s.fNestedPaging)
3847 {
3848 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3849
3850 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3851 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3853 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3854
3855 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3856 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3857 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3858
3859 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3860 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3861 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3862 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3863 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3864 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3865 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3866
3867 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3868 AssertRCReturn(rc, rc);
3869
3870 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3871 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3872 || CPUMIsGuestPagingEnabledEx(pCtx))
3873 {
3874 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3875 if (CPUMIsGuestInPAEModeEx(pCtx))
3876 {
3877 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3878 AssertRCReturn(rc, rc);
3879 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3881 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3883 AssertRCReturn(rc, rc);
3884 }
3885
3886 /*
3887 * The guest's view of its CR3 is unblemished with Nested Paging when the
3888 * guest is using paging or we have unrestricted guest execution to handle
3889 * the guest when it's not using paging.
3890 */
3891 GCPhysGuestCR3 = pCtx->cr3;
3892 }
3893 else
3894 {
3895 /*
3896 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3897 * thinks it accesses physical memory directly, we use our identity-mapped
3898 * page table to map guest-linear to guest-physical addresses. EPT takes care
3899 * of translating it to host-physical addresses.
3900 */
3901 RTGCPHYS GCPhys;
3902 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3903
3904 /* We obtain it here every time as the guest could have relocated this PCI region. */
3905 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3906 if (RT_SUCCESS(rc))
3907 { /* likely */ }
3908 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3909 {
3910 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3911 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3912 }
3913 else
3914 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3915
3916 GCPhysGuestCR3 = GCPhys;
3917 }
3918
3919 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3920 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3921 AssertRCReturn(rc, rc);
3922 }
3923 else
3924 {
3925 /* Non-nested paging case, just use the hypervisor's CR3. */
3926 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3927
3928 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3929 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3930 AssertRCReturn(rc, rc);
3931 }
3932
3933 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3934 }
3935
3936 /*
3937 * Guest CR4.
3938 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3939 */
3940 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3941 {
3942 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
3943 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3944 Assert(!RT_HI_U32(pCtx->cr4));
3945
3946 uint32_t u32GuestCr4 = pCtx->cr4;
3947 uint32_t const u32ShadowCr4 = pCtx->cr4;
3948
3949 /*
3950 * Setup VT-x's view of the guest CR4.
3951 *
3952 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3953 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3954 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3955 *
3956 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3957 */
3958 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3959 {
3960 Assert(pVM->hm.s.vmx.pRealModeTSS);
3961 Assert(PDMVmmDevHeapIsEnabled(pVM));
3962 u32GuestCr4 &= ~X86_CR4_VME;
3963 }
3964
3965 if (pVM->hm.s.fNestedPaging)
3966 {
3967 if ( !CPUMIsGuestPagingEnabledEx(pCtx)
3968 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3969 {
3970 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3971 u32GuestCr4 |= X86_CR4_PSE;
3972 /* Our identity mapping is a 32-bit page directory. */
3973 u32GuestCr4 &= ~X86_CR4_PAE;
3974 }
3975 /* else use guest CR4.*/
3976 }
3977 else
3978 {
3979 /*
3980 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3981 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3982 */
3983 switch (pVCpu->hm.s.enmShadowMode)
3984 {
3985 case PGMMODE_REAL: /* Real-mode. */
3986 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3987 case PGMMODE_32_BIT: /* 32-bit paging. */
3988 {
3989 u32GuestCr4 &= ~X86_CR4_PAE;
3990 break;
3991 }
3992
3993 case PGMMODE_PAE: /* PAE paging. */
3994 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3995 {
3996 u32GuestCr4 |= X86_CR4_PAE;
3997 break;
3998 }
3999
4000 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4001 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4002#ifdef VBOX_ENABLE_64_BITS_GUESTS
4003 break;
4004#endif
4005 default:
4006 AssertFailed();
4007 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4008 }
4009 }
4010
4011 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4012 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4013 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4014 u32GuestCr4 |= fSetCr4;
4015 u32GuestCr4 &= fZapCr4;
4016
4017 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4018 that would cause a VM-exit. */
4019 uint32_t u32Cr4Mask = X86_CR4_VME
4020 | X86_CR4_PAE
4021 | X86_CR4_PGE
4022 | X86_CR4_PSE
4023 | X86_CR4_VMXE;
4024 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4025 u32Cr4Mask |= X86_CR4_OSXSAVE;
4026 if (pVM->cpum.ro.GuestFeatures.fPcid)
4027 u32Cr4Mask |= X86_CR4_PCIDE;
4028
4029 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4030 into the VMCS and update our cache. */
4031 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4032 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4033 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4034 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4035 AssertRCReturn(rc, rc);
4036 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4037
4038 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4039 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
4040
4041 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4042
4043 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4044 fZapCr4));
4045 }
4046 return rc;
4047}
4048
4049
4050/**
4051 * Exports the guest debug registers into the guest-state area in the VMCS.
4052 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4053 *
4054 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4055 *
4056 * @returns VBox status code.
4057 * @param pVCpu The cross context virtual CPU structure.
4058 *
4059 * @remarks No-long-jump zone!!!
4060 */
4061static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu)
4062{
4063 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4064
4065#ifdef VBOX_STRICT
4066 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4067 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
4068 {
4069 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4070 Assert((pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0);
4071 Assert((pVCpu->cpum.GstCtx.dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK);
4072 }
4073#endif
4074
4075 bool fSteppingDB = false;
4076 bool fInterceptMovDRx = false;
4077 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4078 if (pVCpu->hm.s.fSingleInstruction)
4079 {
4080 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4081 PVM pVM = pVCpu->CTX_SUFF(pVM);
4082 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG)
4083 {
4084 uProcCtls |= VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
4085 Assert(fSteppingDB == false);
4086 }
4087 else
4088 {
4089 pVCpu->cpum.GstCtx.eflags.u32 |= X86_EFL_TF;
4090 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4091 pVCpu->hm.s.fClearTrapFlag = true;
4092 fSteppingDB = true;
4093 }
4094 }
4095
4096 uint32_t u32GuestDr7;
4097 if ( fSteppingDB
4098 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4099 {
4100 /*
4101 * Use the combined guest and host DRx values found in the hypervisor register set
4102 * because the debugger has breakpoints active or someone is single stepping on the
4103 * host side without a monitor trap flag.
4104 *
4105 * Note! DBGF expects a clean DR6 state before executing guest code.
4106 */
4107#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4108 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4109 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4110 {
4111 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4112 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4113 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4114 }
4115 else
4116#endif
4117 if (!CPUMIsHyperDebugStateActive(pVCpu))
4118 {
4119 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4120 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4121 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4122 }
4123
4124 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4125 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4126 pVCpu->hm.s.fUsingHyperDR7 = true;
4127 fInterceptMovDRx = true;
4128 }
4129 else
4130 {
4131 /*
4132 * If the guest has enabled debug registers, we need to load them prior to
4133 * executing guest code so they'll trigger at the right time.
4134 */
4135 if (pVCpu->cpum.GstCtx.dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4136 {
4137#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4138 if ( CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx)
4139 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4140 {
4141 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4142 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4143 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4144 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4145 }
4146 else
4147#endif
4148 if (!CPUMIsGuestDebugStateActive(pVCpu))
4149 {
4150 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4151 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4152 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4153 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4154 }
4155 Assert(!fInterceptMovDRx);
4156 }
4157 /*
4158 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4159 * must intercept #DB in order to maintain a correct DR6 guest value, and
4160 * because we need to intercept it to prevent nested #DBs from hanging the
4161 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4162 */
4163#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4164 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4165 && !CPUMIsGuestDebugStateActive(pVCpu))
4166#else
4167 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4168#endif
4169 {
4170 fInterceptMovDRx = true;
4171 }
4172
4173 /* Update DR7 with the actual guest value. */
4174 u32GuestDr7 = pVCpu->cpum.GstCtx.dr[7];
4175 pVCpu->hm.s.fUsingHyperDR7 = false;
4176 }
4177
4178 if (fInterceptMovDRx)
4179 uProcCtls |= VMX_PROC_CTLS_MOV_DR_EXIT;
4180 else
4181 uProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
4182
4183 /*
4184 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4185 * monitor-trap flag and update our cache.
4186 */
4187 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4188 {
4189 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4190 AssertRCReturn(rc2, rc2);
4191 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4192 }
4193
4194 /*
4195 * Update guest DR7.
4196 */
4197 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4198 AssertRCReturn(rc, rc);
4199
4200 return VINF_SUCCESS;
4201}
4202
4203
4204#ifdef VBOX_STRICT
4205/**
4206 * Strict function to validate segment registers.
4207 *
4208 * @param pVCpu The cross context virtual CPU structure.
4209 *
4210 * @remarks Will import guest CR0 on strict builds during validation of
4211 * segments.
4212 */
4213static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu)
4214{
4215 /*
4216 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4217 *
4218 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4219 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4220 * and doesn't change the guest-context value.
4221 */
4222 PVM pVM = pVCpu->CTX_SUFF(pVM);
4223 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4224 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4225 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4226 && ( !CPUMIsGuestInRealModeEx(pCtx)
4227 && !CPUMIsGuestInV86ModeEx(pCtx)))
4228 {
4229 /* Protected mode checks */
4230 /* CS */
4231 Assert(pCtx->cs.Attr.n.u1Present);
4232 Assert(!(pCtx->cs.Attr.u & 0xf00));
4233 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4234 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4235 || !(pCtx->cs.Attr.n.u1Granularity));
4236 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4237 || (pCtx->cs.Attr.n.u1Granularity));
4238 /* CS cannot be loaded with NULL in protected mode. */
4239 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4240 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4241 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4242 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4243 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4244 else
4245 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4246 /* SS */
4247 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4248 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4249 if ( !(pCtx->cr0 & X86_CR0_PE)
4250 || pCtx->cs.Attr.n.u4Type == 3)
4251 {
4252 Assert(!pCtx->ss.Attr.n.u2Dpl);
4253 }
4254 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4255 {
4256 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4257 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4258 Assert(pCtx->ss.Attr.n.u1Present);
4259 Assert(!(pCtx->ss.Attr.u & 0xf00));
4260 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4261 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4262 || !(pCtx->ss.Attr.n.u1Granularity));
4263 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4264 || (pCtx->ss.Attr.n.u1Granularity));
4265 }
4266 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4267 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4268 {
4269 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4270 Assert(pCtx->ds.Attr.n.u1Present);
4271 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4272 Assert(!(pCtx->ds.Attr.u & 0xf00));
4273 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4274 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4275 || !(pCtx->ds.Attr.n.u1Granularity));
4276 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4277 || (pCtx->ds.Attr.n.u1Granularity));
4278 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4279 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4280 }
4281 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4282 {
4283 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4284 Assert(pCtx->es.Attr.n.u1Present);
4285 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4286 Assert(!(pCtx->es.Attr.u & 0xf00));
4287 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4288 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4289 || !(pCtx->es.Attr.n.u1Granularity));
4290 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4291 || (pCtx->es.Attr.n.u1Granularity));
4292 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4293 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4294 }
4295 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->fs.Attr.n.u1Present);
4299 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->fs.Attr.u & 0xf00));
4301 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->fs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4305 || (pCtx->fs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->gs.Attr.n.u1Present);
4313 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->gs.Attr.u & 0xf00));
4315 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->gs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4319 || (pCtx->gs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 /* 64-bit capable CPUs. */
4324# if HC_ARCH_BITS == 64
4325 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4326 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4327 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4328 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4329# endif
4330 }
4331 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4332 || ( CPUMIsGuestInRealModeEx(pCtx)
4333 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4334 {
4335 /* Real and v86 mode checks. */
4336 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4337 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4338 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4339 {
4340 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4341 }
4342 else
4343 {
4344 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4345 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4346 }
4347
4348 /* CS */
4349 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4350 Assert(pCtx->cs.u32Limit == 0xffff);
4351 Assert(u32CSAttr == 0xf3);
4352 /* SS */
4353 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4354 Assert(pCtx->ss.u32Limit == 0xffff);
4355 Assert(u32SSAttr == 0xf3);
4356 /* DS */
4357 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4358 Assert(pCtx->ds.u32Limit == 0xffff);
4359 Assert(u32DSAttr == 0xf3);
4360 /* ES */
4361 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4362 Assert(pCtx->es.u32Limit == 0xffff);
4363 Assert(u32ESAttr == 0xf3);
4364 /* FS */
4365 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4366 Assert(pCtx->fs.u32Limit == 0xffff);
4367 Assert(u32FSAttr == 0xf3);
4368 /* GS */
4369 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4370 Assert(pCtx->gs.u32Limit == 0xffff);
4371 Assert(u32GSAttr == 0xf3);
4372 /* 64-bit capable CPUs. */
4373# if HC_ARCH_BITS == 64
4374 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4375 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4376 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4377 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4378# endif
4379 }
4380}
4381#endif /* VBOX_STRICT */
4382
4383
4384/**
4385 * Exports a guest segment register into the guest-state area in the VMCS.
4386 *
4387 * @returns VBox status code.
4388 * @param pVCpu The cross context virtual CPU structure.
4389 * @param idxSel Index of the selector in the VMCS.
4390 * @param idxLimit Index of the segment limit in the VMCS.
4391 * @param idxBase Index of the segment base in the VMCS.
4392 * @param idxAccess Index of the access rights of the segment in the VMCS.
4393 * @param pSelReg Pointer to the segment selector.
4394 *
4395 * @remarks No-long-jump zone!!!
4396 */
4397static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4398 PCCPUMSELREG pSelReg)
4399{
4400 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4401 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4402 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4403 AssertRCReturn(rc, rc);
4404
4405 uint32_t u32Access = pSelReg->Attr.u;
4406 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4407 {
4408 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4409 u32Access = 0xf3;
4410 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4411 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4412 }
4413 else
4414 {
4415 /*
4416 * The way to differentiate between whether this is really a null selector or was just
4417 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4418 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4419 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4420 * NULL selectors loaded in protected-mode have their attribute as 0.
4421 */
4422 if (!u32Access)
4423 u32Access = X86DESCATTR_UNUSABLE;
4424 }
4425
4426 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4427 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4428 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4429
4430 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4431 AssertRCReturn(rc, rc);
4432 return rc;
4433}
4434
4435
4436/**
4437 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4438 * into the guest-state area in the VMCS.
4439 *
4440 * @returns VBox status code.
4441 * @param pVCpu The cross context virtual CPU structure.
4442 *
4443 * @remarks Will import guest CR0 on strict builds during validation of
4444 * segments.
4445 * @remarks No-long-jump zone!!!
4446 */
4447static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu)
4448{
4449 int rc = VERR_INTERNAL_ERROR_5;
4450 PVM pVM = pVCpu->CTX_SUFF(pVM);
4451 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4452
4453 /*
4454 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4455 */
4456 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4457 {
4458#ifdef VBOX_WITH_REM
4459 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4460 {
4461 Assert(pVM->hm.s.vmx.pRealModeTSS);
4462 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4463 if ( pVCpu->hm.s.vmx.fWasInRealMode
4464 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4465 {
4466 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4467 in real-mode (e.g. OpenBSD 4.0) */
4468 REMFlushTBs(pVM);
4469 Log4Func(("Switch to protected mode detected!\n"));
4470 pVCpu->hm.s.vmx.fWasInRealMode = false;
4471 }
4472 }
4473#endif
4474 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4475 {
4476 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4477 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4478 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pCtx->cs.Attr.u;
4479 rc = HMVMX_EXPORT_SREG(CS, &pCtx->cs);
4480 AssertRCReturn(rc, rc);
4481 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4482 }
4483
4484 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4485 {
4486 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4487 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4488 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pCtx->ss.Attr.u;
4489 rc = HMVMX_EXPORT_SREG(SS, &pCtx->ss);
4490 AssertRCReturn(rc, rc);
4491 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4492 }
4493
4494 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4495 {
4496 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4497 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4498 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pCtx->ds.Attr.u;
4499 rc = HMVMX_EXPORT_SREG(DS, &pCtx->ds);
4500 AssertRCReturn(rc, rc);
4501 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4502 }
4503
4504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4505 {
4506 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4507 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4508 pVCpu->hm.s.vmx.RealMode.AttrES.u = pCtx->es.Attr.u;
4509 rc = HMVMX_EXPORT_SREG(ES, &pCtx->es);
4510 AssertRCReturn(rc, rc);
4511 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4512 }
4513
4514 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4515 {
4516 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4517 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4518 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pCtx->fs.Attr.u;
4519 rc = HMVMX_EXPORT_SREG(FS, &pCtx->fs);
4520 AssertRCReturn(rc, rc);
4521 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4522 }
4523
4524 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4525 {
4526 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4527 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4528 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pCtx->gs.Attr.u;
4529 rc = HMVMX_EXPORT_SREG(GS, &pCtx->gs);
4530 AssertRCReturn(rc, rc);
4531 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4532 }
4533
4534#ifdef VBOX_STRICT
4535 hmR0VmxValidateSegmentRegs(pVCpu);
4536#endif
4537
4538 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pCtx->cs.Sel, pCtx->cs.u64Base,
4539 pCtx->cs.u32Limit, pCtx->cs.Attr.u));
4540 }
4541
4542 /*
4543 * Guest TR.
4544 */
4545 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4546 {
4547 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4548
4549 /*
4550 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4551 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4552 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4553 */
4554 uint16_t u16Sel = 0;
4555 uint32_t u32Limit = 0;
4556 uint64_t u64Base = 0;
4557 uint32_t u32AccessRights = 0;
4558
4559 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4560 {
4561 u16Sel = pCtx->tr.Sel;
4562 u32Limit = pCtx->tr.u32Limit;
4563 u64Base = pCtx->tr.u64Base;
4564 u32AccessRights = pCtx->tr.Attr.u;
4565 }
4566 else
4567 {
4568 Assert(pVM->hm.s.vmx.pRealModeTSS);
4569 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4570
4571 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4572 RTGCPHYS GCPhys;
4573 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4574 AssertRCReturn(rc, rc);
4575
4576 X86DESCATTR DescAttr;
4577 DescAttr.u = 0;
4578 DescAttr.n.u1Present = 1;
4579 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4580
4581 u16Sel = 0;
4582 u32Limit = HM_VTX_TSS_SIZE;
4583 u64Base = GCPhys; /* in real-mode phys = virt. */
4584 u32AccessRights = DescAttr.u;
4585 }
4586
4587 /* Validate. */
4588 Assert(!(u16Sel & RT_BIT(2)));
4589 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4590 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4591 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4592 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4593 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4594 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4595 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4596 Assert( (u32Limit & 0xfff) == 0xfff
4597 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4598 Assert( !(pCtx->tr.u32Limit & 0xfff00000)
4599 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4600
4601 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4603 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4604 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4605 AssertRCReturn(rc, rc);
4606
4607 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4608 Log4Func(("TR base=%#RX64\n", pCtx->tr.u64Base));
4609 }
4610
4611 /*
4612 * Guest GDTR.
4613 */
4614 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4615 {
4616 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4617
4618 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pCtx->gdtr.cbGdt);
4619 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pCtx->gdtr.pGdt);
4620 AssertRCReturn(rc, rc);
4621
4622 /* Validate. */
4623 Assert(!(pCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4624
4625 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4626 Log4Func(("GDTR base=%#RX64\n", pCtx->gdtr.pGdt));
4627 }
4628
4629 /*
4630 * Guest LDTR.
4631 */
4632 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4633 {
4634 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4635
4636 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4637 uint32_t u32Access = 0;
4638 if (!pCtx->ldtr.Attr.u)
4639 u32Access = X86DESCATTR_UNUSABLE;
4640 else
4641 u32Access = pCtx->ldtr.Attr.u;
4642
4643 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pCtx->ldtr.Sel);
4644 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pCtx->ldtr.u32Limit);
4645 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4646 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pCtx->ldtr.u64Base);
4647 AssertRCReturn(rc, rc);
4648
4649 /* Validate. */
4650 if (!(u32Access & X86DESCATTR_UNUSABLE))
4651 {
4652 Assert(!(pCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4653 Assert(pCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4654 Assert(!pCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4655 Assert(pCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4656 Assert(!pCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4657 Assert(!(pCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4658 Assert( (pCtx->ldtr.u32Limit & 0xfff) == 0xfff
4659 || !pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4660 Assert( !(pCtx->ldtr.u32Limit & 0xfff00000)
4661 || pCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4662 }
4663
4664 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4665 Log4Func(("LDTR base=%#RX64\n", pCtx->ldtr.u64Base));
4666 }
4667
4668 /*
4669 * Guest IDTR.
4670 */
4671 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4672 {
4673 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4674
4675 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pCtx->idtr.cbIdt);
4676 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pCtx->idtr.pIdt);
4677 AssertRCReturn(rc, rc);
4678
4679 /* Validate. */
4680 Assert(!(pCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4681
4682 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4683 Log4Func(("IDTR base=%#RX64\n", pCtx->idtr.pIdt));
4684 }
4685
4686 return VINF_SUCCESS;
4687}
4688
4689
4690/**
4691 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4692 * areas.
4693 *
4694 * These MSRs will automatically be loaded to the host CPU on every successful
4695 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4696 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4697 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4698 *
4699 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4700 *
4701 * @returns VBox status code.
4702 * @param pVCpu The cross context virtual CPU structure.
4703 *
4704 * @remarks No-long-jump zone!!!
4705 */
4706static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu)
4707{
4708 AssertPtr(pVCpu);
4709 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4710
4711 /*
4712 * MSRs that we use the auto-load/store MSR area in the VMCS.
4713 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4714 */
4715 PVM pVM = pVCpu->CTX_SUFF(pVM);
4716 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4717 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4718 {
4719 if (pVM->hm.s.fAllow64BitGuests)
4720 {
4721#if HC_ARCH_BITS == 32
4722 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4723
4724 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pCtx->msrLSTAR, false, NULL);
4725 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pCtx->msrSTAR, false, NULL);
4726 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pCtx->msrSFMASK, false, NULL);
4727 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pCtx->msrKERNELGSBASE, false, NULL);
4728 AssertRCReturn(rc, rc);
4729# ifdef LOG_ENABLED
4730 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4731 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4732 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4733# endif
4734#endif
4735 }
4736 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4737 }
4738
4739 /*
4740 * Guest Sysenter MSRs.
4741 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4742 * VM-exits on WRMSRs for these MSRs.
4743 */
4744 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4745 {
4746 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4747
4748 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4749 {
4750 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pCtx->SysEnter.cs);
4751 AssertRCReturn(rc, rc);
4752 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4753 }
4754
4755 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4756 {
4757 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pCtx->SysEnter.eip);
4758 AssertRCReturn(rc, rc);
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4760 }
4761
4762 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4763 {
4764 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pCtx->SysEnter.esp);
4765 AssertRCReturn(rc, rc);
4766 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4767 }
4768 }
4769
4770 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4771 {
4772 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4773
4774 if (hmR0VmxShouldSwapEferMsr(pVCpu))
4775 {
4776 /*
4777 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4778 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4779 */
4780 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4781 {
4782 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pCtx->msrEFER);
4783 AssertRCReturn(rc,rc);
4784 Log4Func(("EFER=%#RX64\n", pCtx->msrEFER));
4785 }
4786 else
4787 {
4788 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pCtx->msrEFER, false /* fUpdateHostMsr */,
4789 NULL /* pfAddedAndUpdated */);
4790 AssertRCReturn(rc, rc);
4791
4792 /* We need to intercept reads too, see @bugref{7386#c16}. */
4793 if (pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_USE_MSR_BITMAPS)
4794 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4795 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pCtx->msrEFER,
4796 pVCpu->hm.s.vmx.cMsrs));
4797 }
4798 }
4799 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4800 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4801 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4802 }
4803
4804 return VINF_SUCCESS;
4805}
4806
4807
4808#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4809/**
4810 * Check if guest state allows safe use of 32-bit switcher again.
4811 *
4812 * Segment bases and protected mode structures must be 32-bit addressable
4813 * because the 32-bit switcher will ignore high dword when writing these VMCS
4814 * fields. See @bugref{8432} for details.
4815 *
4816 * @returns true if safe, false if must continue to use the 64-bit switcher.
4817 * @param pCtx Pointer to the guest-CPU context.
4818 *
4819 * @remarks No-long-jump zone!!!
4820 */
4821static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pCtx)
4822{
4823 if (pCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4824 if (pCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4825 if (pCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4826 if (pCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4827 if (pCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4828 if (pCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4829 if (pCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4830 if (pCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4831 if (pCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4832 if (pCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4833
4834 /* All good, bases are 32-bit. */
4835 return true;
4836}
4837#endif
4838
4839
4840/**
4841 * Selects up the appropriate function to run guest code.
4842 *
4843 * @returns VBox status code.
4844 * @param pVCpu The cross context virtual CPU structure.
4845 *
4846 * @remarks No-long-jump zone!!!
4847 */
4848static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu)
4849{
4850 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4851 if (CPUMIsGuestInLongModeEx(pCtx))
4852 {
4853#ifndef VBOX_ENABLE_64_BITS_GUESTS
4854 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4855#endif
4856 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4857#if HC_ARCH_BITS == 32
4858 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4859 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4860 {
4861#ifdef VBOX_STRICT
4862 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4863 {
4864 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4865 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4866 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4867 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4868 | HM_CHANGED_VMX_ENTRY_CTLS
4869 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4870 }
4871#endif
4872 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4873
4874 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4875 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4876 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4877 Log4Func(("Selected 64-bit switcher\n"));
4878 }
4879#else
4880 /* 64-bit host. */
4881 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4882#endif
4883 }
4884 else
4885 {
4886 /* Guest is not in long mode, use the 32-bit handler. */
4887#if HC_ARCH_BITS == 32
4888 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4889 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4890 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4891 {
4892# ifdef VBOX_STRICT
4893 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4894 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4895 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4896 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4897 | HM_CHANGED_VMX_ENTRY_CTLS
4898 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4899# endif
4900 }
4901# ifdef VBOX_ENABLE_64_BITS_GUESTS
4902 /*
4903 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4904 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4905 * switcher flag because now we know the guest is in a sane state where it's safe
4906 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4907 * the much faster 32-bit switcher again.
4908 */
4909 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4910 {
4911 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4912 Log4Func(("Selected 32-bit switcher\n"));
4913 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4914 }
4915 else
4916 {
4917 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4918 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4919 || hmR0VmxIs32BitSwitcherSafe(pCtx))
4920 {
4921 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4922 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4923 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4924 | HM_CHANGED_VMX_ENTRY_CTLS
4925 | HM_CHANGED_VMX_EXIT_CTLS
4926 | HM_CHANGED_HOST_CONTEXT);
4927 Log4Func(("Selected 32-bit switcher (safe)\n"));
4928 }
4929 }
4930# else
4931 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4932# endif
4933#else
4934 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4935#endif
4936 }
4937 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4938 return VINF_SUCCESS;
4939}
4940
4941
4942/**
4943 * Wrapper for running the guest code in VT-x.
4944 *
4945 * @returns VBox status code, no informational status codes.
4946 * @param pVCpu The cross context virtual CPU structure.
4947 *
4948 * @remarks No-long-jump zone!!!
4949 */
4950DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu)
4951{
4952 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4953 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
4954 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4955
4956 /*
4957 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4958 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4959 * callee-saved and thus the need for this XMM wrapper.
4960 *
4961 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4962 */
4963 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4964 /** @todo Add stats for resume vs launch. */
4965 PVM pVM = pVCpu->CTX_SUFF(pVM);
4966#ifdef VBOX_WITH_KERNEL_USING_XMM
4967 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4968#else
4969 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4970#endif
4971 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4972 return rc;
4973}
4974
4975
4976/**
4977 * Reports world-switch error and dumps some useful debug info.
4978 *
4979 * @param pVCpu The cross context virtual CPU structure.
4980 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4981 * @param pVmxTransient Pointer to the VMX transient structure (only
4982 * exitReason updated).
4983 */
4984static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PVMXTRANSIENT pVmxTransient)
4985{
4986 Assert(pVCpu);
4987 Assert(pVmxTransient);
4988 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
4989
4990 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4991 switch (rcVMRun)
4992 {
4993 case VERR_VMX_INVALID_VMXON_PTR:
4994 AssertFailed();
4995 break;
4996 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4997 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4998 {
4999 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5000 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5001 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5002 AssertRC(rc);
5003
5004 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5005 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5006 Cannot do it here as we may have been long preempted. */
5007
5008#ifdef VBOX_STRICT
5009 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5010 pVmxTransient->uExitReason));
5011 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5012 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5013 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5014 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5015 else
5016 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5017 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5018 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5019
5020 /* VMX control bits. */
5021 uint32_t u32Val;
5022 uint64_t u64Val;
5023 RTHCUINTREG uHCReg;
5024 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5025 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5028 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_SECONDARY_CTLS)
5029 {
5030 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5031 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5032 }
5033 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5034 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5035 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5036 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5045 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5046 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5047 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5048 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5049 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5050 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5057 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5058 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5060 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5061 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5062 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5063 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5064 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5065 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5066 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5067 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5068 {
5069 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5070 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5071 }
5072
5073 /* Guest bits. */
5074 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5075 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rip, u64Val));
5076 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5077 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pVCpu->cpum.GstCtx.rsp, u64Val));
5078 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5079 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pVCpu->cpum.GstCtx.eflags.u32, u32Val));
5080 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5081 {
5082 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5084 }
5085
5086 /* Host bits. */
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5088 Log4(("Host CR0 %#RHr\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5090 Log4(("Host CR3 %#RHr\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5092 Log4(("Host CR4 %#RHr\n", uHCReg));
5093
5094 RTGDTR HostGdtr;
5095 PCX86DESCHC pDesc;
5096 ASMGetGDTR(&HostGdtr);
5097 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5098 Log4(("Host CS %#08x\n", u32Val));
5099 if (u32Val < HostGdtr.cbGdt)
5100 {
5101 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5102 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5103 }
5104
5105 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5106 Log4(("Host DS %#08x\n", u32Val));
5107 if (u32Val < HostGdtr.cbGdt)
5108 {
5109 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5110 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5111 }
5112
5113 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5114 Log4(("Host ES %#08x\n", u32Val));
5115 if (u32Val < HostGdtr.cbGdt)
5116 {
5117 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5118 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5119 }
5120
5121 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5122 Log4(("Host FS %#08x\n", u32Val));
5123 if (u32Val < HostGdtr.cbGdt)
5124 {
5125 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5126 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5127 }
5128
5129 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5130 Log4(("Host GS %#08x\n", u32Val));
5131 if (u32Val < HostGdtr.cbGdt)
5132 {
5133 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5134 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5135 }
5136
5137 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5138 Log4(("Host SS %#08x\n", u32Val));
5139 if (u32Val < HostGdtr.cbGdt)
5140 {
5141 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5142 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5143 }
5144
5145 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5146 Log4(("Host TR %#08x\n", u32Val));
5147 if (u32Val < HostGdtr.cbGdt)
5148 {
5149 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5150 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5151 }
5152
5153 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5154 Log4(("Host TR Base %#RHv\n", uHCReg));
5155 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5156 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5157 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5158 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5159 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5160 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5161 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5162 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5164 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5165 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5166 Log4(("Host RSP %#RHv\n", uHCReg));
5167 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5168 Log4(("Host RIP %#RHv\n", uHCReg));
5169# if HC_ARCH_BITS == 64
5170 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5171 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5172 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5173 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5174 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5175 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5176# endif
5177#endif /* VBOX_STRICT */
5178 break;
5179 }
5180
5181 default:
5182 /* Impossible */
5183 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5184 break;
5185 }
5186}
5187
5188
5189#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5190#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5191# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5192#endif
5193#ifdef VBOX_STRICT
5194static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5195{
5196 switch (idxField)
5197 {
5198 case VMX_VMCS_GUEST_RIP:
5199 case VMX_VMCS_GUEST_RSP:
5200 case VMX_VMCS_GUEST_SYSENTER_EIP:
5201 case VMX_VMCS_GUEST_SYSENTER_ESP:
5202 case VMX_VMCS_GUEST_GDTR_BASE:
5203 case VMX_VMCS_GUEST_IDTR_BASE:
5204 case VMX_VMCS_GUEST_CS_BASE:
5205 case VMX_VMCS_GUEST_DS_BASE:
5206 case VMX_VMCS_GUEST_ES_BASE:
5207 case VMX_VMCS_GUEST_FS_BASE:
5208 case VMX_VMCS_GUEST_GS_BASE:
5209 case VMX_VMCS_GUEST_SS_BASE:
5210 case VMX_VMCS_GUEST_LDTR_BASE:
5211 case VMX_VMCS_GUEST_TR_BASE:
5212 case VMX_VMCS_GUEST_CR3:
5213 return true;
5214 }
5215 return false;
5216}
5217
5218static bool hmR0VmxIsValidReadField(uint32_t idxField)
5219{
5220 switch (idxField)
5221 {
5222 /* Read-only fields. */
5223 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5224 return true;
5225 }
5226 /* Remaining readable fields should also be writable. */
5227 return hmR0VmxIsValidWriteField(idxField);
5228}
5229#endif /* VBOX_STRICT */
5230
5231
5232/**
5233 * Executes the specified handler in 64-bit mode.
5234 *
5235 * @returns VBox status code (no informational status codes).
5236 * @param pVCpu The cross context virtual CPU structure.
5237 * @param enmOp The operation to perform.
5238 * @param cParams Number of parameters.
5239 * @param paParam Array of 32-bit parameters.
5240 */
5241VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5242{
5243 PVM pVM = pVCpu->CTX_SUFF(pVM);
5244 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5245 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5246 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5247 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5248
5249#ifdef VBOX_STRICT
5250 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5251 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5252
5253 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5254 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5255#endif
5256
5257 /* Disable interrupts. */
5258 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5259
5260#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5261 RTCPUID idHostCpu = RTMpCpuId();
5262 CPUMR0SetLApic(pVCpu, idHostCpu);
5263#endif
5264
5265 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5266 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5267
5268 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5269 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5270 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5271
5272 /* Leave VMX Root Mode. */
5273 VMXDisable();
5274
5275 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5276
5277 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5278 CPUMSetHyperEIP(pVCpu, enmOp);
5279 for (int i = (int)cParams - 1; i >= 0; i--)
5280 CPUMPushHyper(pVCpu, paParam[i]);
5281
5282 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5283
5284 /* Call the switcher. */
5285 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_UOFFSETOF_DYN(VM, aCpus[pVCpu->idCpu].cpum) - RT_UOFFSETOF(VM, cpum));
5286 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5287
5288 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5289 /* Make sure the VMX instructions don't cause #UD faults. */
5290 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5291
5292 /* Re-enter VMX Root Mode */
5293 int rc2 = VMXEnable(HCPhysCpuPage);
5294 if (RT_FAILURE(rc2))
5295 {
5296 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5297 ASMSetFlags(fOldEFlags);
5298 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5299 return rc2;
5300 }
5301
5302 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5303 AssertRC(rc2);
5304 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5305 Assert(!(ASMGetFlags() & X86_EFL_IF));
5306 ASMSetFlags(fOldEFlags);
5307 return rc;
5308}
5309
5310
5311/**
5312 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5313 * supporting 64-bit guests.
5314 *
5315 * @returns VBox status code.
5316 * @param fResume Whether to VMLAUNCH or VMRESUME.
5317 * @param pCtx Pointer to the guest-CPU context.
5318 * @param pCache Pointer to the VMCS cache.
5319 * @param pVM The cross context VM structure.
5320 * @param pVCpu The cross context virtual CPU structure.
5321 */
5322DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5323{
5324 NOREF(fResume);
5325
5326 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5327 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5328
5329#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5330 pCache->uPos = 1;
5331 pCache->interPD = PGMGetInterPaeCR3(pVM);
5332 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5333#endif
5334
5335#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5336 pCache->TestIn.HCPhysCpuPage = 0;
5337 pCache->TestIn.HCPhysVmcs = 0;
5338 pCache->TestIn.pCache = 0;
5339 pCache->TestOut.HCPhysVmcs = 0;
5340 pCache->TestOut.pCache = 0;
5341 pCache->TestOut.pCtx = 0;
5342 pCache->TestOut.eflags = 0;
5343#else
5344 NOREF(pCache);
5345#endif
5346
5347 uint32_t aParam[10];
5348 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5349 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5350 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5351 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5352 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5353 aParam[5] = 0;
5354 aParam[6] = VM_RC_ADDR(pVM, pVM);
5355 aParam[7] = 0;
5356 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5357 aParam[9] = 0;
5358
5359#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5360 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5361 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5362#endif
5363 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5364
5365#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5366 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5367 Assert(pCtx->dr[4] == 10);
5368 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5369#endif
5370
5371#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5372 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5373 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5374 pVCpu->hm.s.vmx.HCPhysVmcs));
5375 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5376 pCache->TestOut.HCPhysVmcs));
5377 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5378 pCache->TestOut.pCache));
5379 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5380 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5381 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5382 pCache->TestOut.pCtx));
5383 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5384#endif
5385 NOREF(pCtx);
5386 return rc;
5387}
5388
5389
5390/**
5391 * Initialize the VMCS-Read cache.
5392 *
5393 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5394 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5395 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5396 * (those that have a 32-bit FULL & HIGH part).
5397 *
5398 * @returns VBox status code.
5399 * @param pVCpu The cross context virtual CPU structure.
5400 */
5401static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5402{
5403#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5404 do { \
5405 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5406 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5407 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5408 ++cReadFields; \
5409 } while (0)
5410
5411 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5412 uint32_t cReadFields = 0;
5413
5414 /*
5415 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5416 * and serve to indicate exceptions to the rules.
5417 */
5418
5419 /* Guest-natural selector base fields. */
5420#if 0
5421 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5424#endif
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5434 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5435 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5437#if 0
5438 /* Unused natural width guest-state fields. */
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5441#endif
5442 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5443 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5444
5445 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5446 these 64-bit fields (using "FULL" and "HIGH" fields). */
5447#if 0
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5457#endif
5458
5459 /* Natural width guest-state fields. */
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5461#if 0
5462 /* Currently unused field. */
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5464#endif
5465
5466 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5467 {
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5469 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5470 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5471 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5472 }
5473 else
5474 {
5475 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5476 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5477 }
5478
5479#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5480 return VINF_SUCCESS;
5481}
5482
5483
5484/**
5485 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5486 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5487 * darwin, running 64-bit guests).
5488 *
5489 * @returns VBox status code.
5490 * @param pVCpu The cross context virtual CPU structure.
5491 * @param idxField The VMCS field encoding.
5492 * @param u64Val 16, 32 or 64-bit value.
5493 */
5494VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5495{
5496 int rc;
5497 switch (idxField)
5498 {
5499 /*
5500 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5501 */
5502 /* 64-bit Control fields. */
5503 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5504 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5505 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5506 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5507 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5508 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5509 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5510 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5511 case VMX_VMCS64_CTRL_VIRT_APIC_PAGEADDR_FULL:
5512 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5513 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5514 case VMX_VMCS64_CTRL_EPTP_FULL:
5515 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5516 /* 64-bit Guest-state fields. */
5517 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5518 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5519 case VMX_VMCS64_GUEST_PAT_FULL:
5520 case VMX_VMCS64_GUEST_EFER_FULL:
5521 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5522 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5523 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5524 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5525 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5526 /* 64-bit Host-state fields. */
5527 case VMX_VMCS64_HOST_PAT_FULL:
5528 case VMX_VMCS64_HOST_EFER_FULL:
5529 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5530 {
5531 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5532 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5533 break;
5534 }
5535
5536 /*
5537 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5538 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5539 */
5540 /* Natural-width Guest-state fields. */
5541 case VMX_VMCS_GUEST_CR3:
5542 case VMX_VMCS_GUEST_ES_BASE:
5543 case VMX_VMCS_GUEST_CS_BASE:
5544 case VMX_VMCS_GUEST_SS_BASE:
5545 case VMX_VMCS_GUEST_DS_BASE:
5546 case VMX_VMCS_GUEST_FS_BASE:
5547 case VMX_VMCS_GUEST_GS_BASE:
5548 case VMX_VMCS_GUEST_LDTR_BASE:
5549 case VMX_VMCS_GUEST_TR_BASE:
5550 case VMX_VMCS_GUEST_GDTR_BASE:
5551 case VMX_VMCS_GUEST_IDTR_BASE:
5552 case VMX_VMCS_GUEST_RSP:
5553 case VMX_VMCS_GUEST_RIP:
5554 case VMX_VMCS_GUEST_SYSENTER_ESP:
5555 case VMX_VMCS_GUEST_SYSENTER_EIP:
5556 {
5557 if (!(RT_HI_U32(u64Val)))
5558 {
5559 /* If this field is 64-bit, VT-x will zero out the top bits. */
5560 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5561 }
5562 else
5563 {
5564 /* Assert that only the 32->64 switcher case should ever come here. */
5565 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5566 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5567 }
5568 break;
5569 }
5570
5571 default:
5572 {
5573 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5574 rc = VERR_INVALID_PARAMETER;
5575 break;
5576 }
5577 }
5578 AssertRCReturn(rc, rc);
5579 return rc;
5580}
5581
5582
5583/**
5584 * Queue up a VMWRITE by using the VMCS write cache.
5585 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5586 *
5587 * @param pVCpu The cross context virtual CPU structure.
5588 * @param idxField The VMCS field encoding.
5589 * @param u64Val 16, 32 or 64-bit value.
5590 */
5591VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5592{
5593 AssertPtr(pVCpu);
5594 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5595
5596 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5597 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5598
5599 /* Make sure there are no duplicates. */
5600 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5601 {
5602 if (pCache->Write.aField[i] == idxField)
5603 {
5604 pCache->Write.aFieldVal[i] = u64Val;
5605 return VINF_SUCCESS;
5606 }
5607 }
5608
5609 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5610 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5611 pCache->Write.cValidEntries++;
5612 return VINF_SUCCESS;
5613}
5614#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5615
5616
5617/**
5618 * Sets up the usage of TSC-offsetting and updates the VMCS.
5619 *
5620 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5621 * VMX preemption timer.
5622 *
5623 * @returns VBox status code.
5624 * @param pVCpu The cross context virtual CPU structure.
5625 *
5626 * @remarks No-long-jump zone!!!
5627 */
5628static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5629{
5630 bool fOffsettedTsc;
5631 bool fParavirtTsc;
5632 PVM pVM = pVCpu->CTX_SUFF(pVM);
5633 uint64_t uTscOffset;
5634 if (pVM->hm.s.vmx.fUsePreemptTimer)
5635 {
5636 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5637
5638 /* Make sure the returned values have sane upper and lower boundaries. */
5639 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5640 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5641 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5642 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5643
5644 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5645 int rc = VMXWriteVmcs32(VMX_VMCS32_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5646 AssertRC(rc);
5647 }
5648 else
5649 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5650
5651 if (fParavirtTsc)
5652 {
5653 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5654 information before every VM-entry, hence disable it for performance sake. */
5655#if 0
5656 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5657 AssertRC(rc);
5658#endif
5659 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5660 }
5661
5662 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5663 if ( fOffsettedTsc
5664 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5665 {
5666 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5667 {
5668 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5669 AssertRC(rc);
5670 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5671 }
5672
5673 if (uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT)
5674 {
5675 uProcCtls &= ~VMX_PROC_CTLS_RDTSC_EXIT;
5676 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5677 AssertRC(rc);
5678 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5679 }
5680 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5681 }
5682 else
5683 {
5684 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5685 if (!(uProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
5686 {
5687 uProcCtls |= VMX_PROC_CTLS_RDTSC_EXIT;
5688 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5689 AssertRC(rc);
5690 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5691 }
5692 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5693 }
5694}
5695
5696
5697/**
5698 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5699 * VM-exit interruption info type.
5700 *
5701 * @returns The IEM exception flags.
5702 * @param uVector The event vector.
5703 * @param uVmxVectorType The VMX event type.
5704 *
5705 * @remarks This function currently only constructs flags required for
5706 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5707 * and CR2 aspects of an exception are not included).
5708 */
5709static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5710{
5711 uint32_t fIemXcptFlags;
5712 switch (uVmxVectorType)
5713 {
5714 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5715 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5716 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5717 break;
5718
5719 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5720 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5721 break;
5722
5723 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5724 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5725 break;
5726
5727 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5728 {
5729 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5730 if (uVector == X86_XCPT_BP)
5731 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5732 else if (uVector == X86_XCPT_OF)
5733 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5734 else
5735 {
5736 fIemXcptFlags = 0;
5737 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5738 }
5739 break;
5740 }
5741
5742 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5743 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5744 break;
5745
5746 default:
5747 fIemXcptFlags = 0;
5748 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5749 break;
5750 }
5751 return fIemXcptFlags;
5752}
5753
5754
5755/**
5756 * Sets an event as a pending event to be injected into the guest.
5757 *
5758 * @param pVCpu The cross context virtual CPU structure.
5759 * @param u32IntInfo The VM-entry interruption-information field.
5760 * @param cbInstr The VM-entry instruction length in bytes (for software
5761 * interrupts, exceptions and privileged software
5762 * exceptions).
5763 * @param u32ErrCode The VM-entry exception error code.
5764 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5765 * page-fault.
5766 *
5767 * @remarks Statistics counter assumes this is a guest event being injected or
5768 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5769 * always incremented.
5770 */
5771DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5772 RTGCUINTPTR GCPtrFaultAddress)
5773{
5774 Assert(!pVCpu->hm.s.Event.fPending);
5775 pVCpu->hm.s.Event.fPending = true;
5776 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5777 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5778 pVCpu->hm.s.Event.cbInstr = cbInstr;
5779 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5780}
5781
5782
5783/**
5784 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5785 *
5786 * @param pVCpu The cross context virtual CPU structure.
5787 */
5788DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu)
5789{
5790 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
5791 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
5792 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
5793 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5794}
5795
5796
5797/**
5798 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
5799 *
5800 * @param pVCpu The cross context virtual CPU structure.
5801 */
5802DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu)
5803{
5804 uint32_t const u32IntInfo = X86_XCPT_UD | VMX_EXIT_INT_INFO_VALID
5805 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
5806 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5807}
5808
5809
5810/**
5811 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
5812 *
5813 * @param pVCpu The cross context virtual CPU structure.
5814 */
5815DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu)
5816{
5817 uint32_t const u32IntInfo = X86_XCPT_DB | VMX_EXIT_INT_INFO_VALID
5818 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
5819 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5820}
5821
5822
5823/**
5824 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
5825 *
5826 * @param pVCpu The cross context virtual CPU structure.
5827 * @param cbInstr The value of RIP that is to be pushed on the guest
5828 * stack.
5829 */
5830DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, uint32_t cbInstr)
5831{
5832 uint32_t const u32IntInfo = X86_XCPT_OF | VMX_EXIT_INT_INFO_VALID
5833 | (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
5834 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5835}
5836
5837
5838/**
5839 * Handle a condition that occurred while delivering an event through the guest
5840 * IDT.
5841 *
5842 * @returns Strict VBox status code (i.e. informational status codes too).
5843 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5844 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5845 * to continue execution of the guest which will delivery the \#DF.
5846 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5847 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5848 *
5849 * @param pVCpu The cross context virtual CPU structure.
5850 * @param pVmxTransient Pointer to the VMX transient structure.
5851 *
5852 * @remarks No-long-jump zone!!!
5853 */
5854static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
5855{
5856 uint32_t const uExitVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5857
5858 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5859 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5860 AssertRCReturn(rc2, rc2);
5861
5862 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5863 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5864 {
5865 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5866 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5867
5868 /*
5869 * If the event was a software interrupt (generated with INT n) or a software exception
5870 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5871 * can handle the VM-exit and continue guest execution which will re-execute the
5872 * instruction rather than re-injecting the exception, as that can cause premature
5873 * trips to ring-3 before injection and involve TRPM which currently has no way of
5874 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5875 * the problem).
5876 */
5877 IEMXCPTRAISE enmRaise;
5878 IEMXCPTRAISEINFO fRaiseInfo;
5879 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5880 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5881 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5882 {
5883 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5884 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5885 }
5886 else if (VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5887 {
5888 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5889 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5890 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5891 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5892 AssertMsgReturn(uExitVectorType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT,
5893 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5894 uExitVectorType), VERR_VMX_IPE_5);
5895
5896 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5897
5898 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5899 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5900 {
5901 pVmxTransient->fVectoringPF = true;
5902 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5903 }
5904 }
5905 else
5906 {
5907 /*
5908 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5909 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5910 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5911 */
5912 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5913 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5914 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5915 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5916 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5917 }
5918
5919 /*
5920 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5921 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5922 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5923 * subsequent VM-entry would fail.
5924 *
5925 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5926 */
5927 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5928 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5929 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5930 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5931 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
5932 {
5933 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5934 }
5935
5936 switch (enmRaise)
5937 {
5938 case IEMXCPTRAISE_CURRENT_XCPT:
5939 {
5940 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5941 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5942 Assert(rcStrict == VINF_SUCCESS);
5943 break;
5944 }
5945
5946 case IEMXCPTRAISE_PREV_EVENT:
5947 {
5948 uint32_t u32ErrCode;
5949 if (VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo))
5950 {
5951 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5952 AssertRCReturn(rc2, rc2);
5953 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5954 }
5955 else
5956 u32ErrCode = 0;
5957
5958 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5959 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5960 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5961 0 /* cbInstr */, u32ErrCode, pVCpu->cpum.GstCtx.cr2);
5962
5963 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5964 pVCpu->hm.s.Event.u32ErrCode));
5965 Assert(rcStrict == VINF_SUCCESS);
5966 break;
5967 }
5968
5969 case IEMXCPTRAISE_REEXEC_INSTR:
5970 Assert(rcStrict == VINF_SUCCESS);
5971 break;
5972
5973 case IEMXCPTRAISE_DOUBLE_FAULT:
5974 {
5975 /*
5976 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5977 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5978 */
5979 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5980 {
5981 pVmxTransient->fVectoringDoublePF = true;
5982 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5983 pVCpu->cpum.GstCtx.cr2));
5984 rcStrict = VINF_SUCCESS;
5985 }
5986 else
5987 {
5988 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5989 hmR0VmxSetPendingXcptDF(pVCpu);
5990 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5991 uIdtVector, uExitVector));
5992 rcStrict = VINF_HM_DOUBLE_FAULT;
5993 }
5994 break;
5995 }
5996
5997 case IEMXCPTRAISE_TRIPLE_FAULT:
5998 {
5999 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6000 rcStrict = VINF_EM_RESET;
6001 break;
6002 }
6003
6004 case IEMXCPTRAISE_CPU_HANG:
6005 {
6006 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6007 rcStrict = VERR_EM_GUEST_CPU_HANG;
6008 break;
6009 }
6010
6011 default:
6012 {
6013 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6014 rcStrict = VERR_VMX_IPE_2;
6015 break;
6016 }
6017 }
6018 }
6019 else if ( VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6020 && VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6021 && uExitVector != X86_XCPT_DF
6022 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI))
6023 {
6024 /*
6025 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6026 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6027 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6028 */
6029 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6030 {
6031 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6032 VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6033 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6034 }
6035 }
6036
6037 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6038 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6039 return rcStrict;
6040}
6041
6042
6043/**
6044 * Imports a guest segment register from the current VMCS into
6045 * the guest-CPU context.
6046 *
6047 * @returns VBox status code.
6048 * @param pVCpu The cross context virtual CPU structure.
6049 * @param idxSel Index of the selector in the VMCS.
6050 * @param idxLimit Index of the segment limit in the VMCS.
6051 * @param idxBase Index of the segment base in the VMCS.
6052 * @param idxAccess Index of the access rights of the segment in the VMCS.
6053 * @param pSelReg Pointer to the segment selector.
6054 *
6055 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6056 * do not log!
6057 *
6058 * @remarks Never call this function directly!!! Use the
6059 * HMVMX_IMPORT_SREG() macro as that takes care
6060 * of whether to read from the VMCS cache or not.
6061 */
6062static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6063 PCPUMSELREG pSelReg)
6064{
6065 NOREF(pVCpu);
6066
6067 uint32_t u32Sel;
6068 uint32_t u32Limit;
6069 uint32_t u32Attr;
6070 uint64_t u64Base;
6071 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6072 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6073 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6074 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6075 AssertRCReturn(rc, rc);
6076
6077 pSelReg->Sel = (uint16_t)u32Sel;
6078 pSelReg->ValidSel = (uint16_t)u32Sel;
6079 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6080 pSelReg->u32Limit = u32Limit;
6081 pSelReg->u64Base = u64Base;
6082 pSelReg->Attr.u = u32Attr;
6083
6084 /*
6085 * If VT-x marks the segment as unusable, most other bits remain undefined:
6086 * - For CS the L, D and G bits have meaning.
6087 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6088 * - For the remaining data segments no bits are defined.
6089 *
6090 * The present bit and the unusable bit has been observed to be set at the
6091 * same time (the selector was supposed to be invalid as we started executing
6092 * a V8086 interrupt in ring-0).
6093 *
6094 * What should be important for the rest of the VBox code, is that the P bit is
6095 * cleared. Some of the other VBox code recognizes the unusable bit, but
6096 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6097 * safe side here, we'll strip off P and other bits we don't care about. If
6098 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6099 *
6100 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6101 */
6102 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6103 {
6104 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6105
6106 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6107 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6108 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6109#ifdef VBOX_STRICT
6110 VMMRZCallRing3Disable(pVCpu);
6111 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6112# ifdef DEBUG_bird
6113 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6114 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6115 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6116# endif
6117 VMMRZCallRing3Enable(pVCpu);
6118#endif
6119 }
6120 return VINF_SUCCESS;
6121}
6122
6123
6124/**
6125 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6126 *
6127 * @returns VBox status code.
6128 * @param pVCpu The cross context virtual CPU structure.
6129 *
6130 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6131 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6132 * instead!!!
6133 */
6134DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6135{
6136 uint64_t u64Val;
6137 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6138 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6139 {
6140 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6141 if (RT_SUCCESS(rc))
6142 {
6143 pCtx->rip = u64Val;
6144 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6145 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6146 }
6147 return rc;
6148 }
6149 return VINF_SUCCESS;
6150}
6151
6152
6153/**
6154 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6155 *
6156 * @returns VBox status code.
6157 * @param pVCpu The cross context virtual CPU structure.
6158 *
6159 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6160 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6161 * instead!!!
6162 */
6163DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6164{
6165 uint32_t u32Val;
6166 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6167 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6168 {
6169 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6170 if (RT_SUCCESS(rc))
6171 {
6172 pCtx->eflags.u32 = u32Val;
6173
6174 /* Restore eflags for real-on-v86-mode hack. */
6175 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6176 {
6177 pCtx->eflags.Bits.u1VM = 0;
6178 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6179 }
6180 }
6181 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6182 return rc;
6183 }
6184 return VINF_SUCCESS;
6185}
6186
6187
6188/**
6189 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6190 * context.
6191 *
6192 * @returns VBox status code.
6193 * @param pVCpu The cross context virtual CPU structure.
6194 *
6195 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6196 * do not log!
6197 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6198 * instead!!!
6199 */
6200DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6201{
6202 uint32_t u32Val;
6203 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6204 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32Val);
6205 if (RT_SUCCESS(rc))
6206 {
6207 /*
6208 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6209 * might need them in hmR0VmxEvaluatePendingEvent().
6210 */
6211 if (!u32Val)
6212 {
6213 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6214 {
6215 rc = hmR0VmxImportGuestRip(pVCpu);
6216 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6217 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6218 }
6219
6220 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6221 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6222 }
6223 else
6224 {
6225 rc = hmR0VmxImportGuestRip(pVCpu);
6226 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6227
6228 if (u32Val & ( VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
6229 | VMX_VMCS_GUEST_INT_STATE_BLOCK_STI))
6230 {
6231 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6232 }
6233 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6234 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6235
6236 if (u32Val & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI)
6237 {
6238 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6239 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6240 }
6241 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6242 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6243 }
6244 }
6245 return rc;
6246}
6247
6248
6249/**
6250 * Worker for VMXR0ImportStateOnDemand.
6251 *
6252 * @returns VBox status code.
6253 * @param pVCpu The cross context virtual CPU structure.
6254 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6255 */
6256static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6257{
6258#define VMXLOCAL_BREAK_RC(a_rc) \
6259 if (RT_FAILURE(a_rc)) \
6260 break
6261
6262 int rc = VINF_SUCCESS;
6263 PVM pVM = pVCpu->CTX_SUFF(pVM);
6264 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6265 uint64_t u64Val;
6266 uint32_t u32Val;
6267
6268 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6269 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6270
6271 /*
6272 * We disable interrupts to make the updating of the state and in particular
6273 * the fExtrn modification atomic wrt to preemption hooks.
6274 */
6275 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6276
6277 fWhat &= pCtx->fExtrn;
6278 if (fWhat)
6279 {
6280 do
6281 {
6282 if (fWhat & CPUMCTX_EXTRN_RIP)
6283 {
6284 rc = hmR0VmxImportGuestRip(pVCpu);
6285 VMXLOCAL_BREAK_RC(rc);
6286 }
6287
6288 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6289 {
6290 rc = hmR0VmxImportGuestRFlags(pVCpu);
6291 VMXLOCAL_BREAK_RC(rc);
6292 }
6293
6294 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6295 {
6296 rc = hmR0VmxImportGuestIntrState(pVCpu);
6297 VMXLOCAL_BREAK_RC(rc);
6298 }
6299
6300 if (fWhat & CPUMCTX_EXTRN_RSP)
6301 {
6302 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6303 VMXLOCAL_BREAK_RC(rc);
6304 pCtx->rsp = u64Val;
6305 }
6306
6307 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6308 {
6309 if (fWhat & CPUMCTX_EXTRN_CS)
6310 {
6311 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6312 rc |= hmR0VmxImportGuestRip(pVCpu);
6313 VMXLOCAL_BREAK_RC(rc);
6314 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6315 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6316 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6317 }
6318 if (fWhat & CPUMCTX_EXTRN_SS)
6319 {
6320 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6321 VMXLOCAL_BREAK_RC(rc);
6322 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6323 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6324 }
6325 if (fWhat & CPUMCTX_EXTRN_DS)
6326 {
6327 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6328 VMXLOCAL_BREAK_RC(rc);
6329 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6330 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6331 }
6332 if (fWhat & CPUMCTX_EXTRN_ES)
6333 {
6334 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6335 VMXLOCAL_BREAK_RC(rc);
6336 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6337 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6338 }
6339 if (fWhat & CPUMCTX_EXTRN_FS)
6340 {
6341 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6342 VMXLOCAL_BREAK_RC(rc);
6343 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6344 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6345 }
6346 if (fWhat & CPUMCTX_EXTRN_GS)
6347 {
6348 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6349 VMXLOCAL_BREAK_RC(rc);
6350 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6351 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6352 }
6353 }
6354
6355 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6356 {
6357 if (fWhat & CPUMCTX_EXTRN_LDTR)
6358 {
6359 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6360 VMXLOCAL_BREAK_RC(rc);
6361 }
6362
6363 if (fWhat & CPUMCTX_EXTRN_GDTR)
6364 {
6365 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6366 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6367 VMXLOCAL_BREAK_RC(rc);
6368 pCtx->gdtr.pGdt = u64Val;
6369 pCtx->gdtr.cbGdt = u32Val;
6370 }
6371
6372 /* Guest IDTR. */
6373 if (fWhat & CPUMCTX_EXTRN_IDTR)
6374 {
6375 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6376 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6377 VMXLOCAL_BREAK_RC(rc);
6378 pCtx->idtr.pIdt = u64Val;
6379 pCtx->idtr.cbIdt = u32Val;
6380 }
6381
6382 /* Guest TR. */
6383 if (fWhat & CPUMCTX_EXTRN_TR)
6384 {
6385 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6386 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6387 {
6388 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6389 VMXLOCAL_BREAK_RC(rc);
6390 }
6391 }
6392 }
6393
6394 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6395 {
6396 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6397 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6398 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6399 pCtx->SysEnter.cs = u32Val;
6400 VMXLOCAL_BREAK_RC(rc);
6401 }
6402
6403#if HC_ARCH_BITS == 64
6404 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6405 {
6406 if ( pVM->hm.s.fAllow64BitGuests
6407 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6408 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6409 }
6410
6411 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6412 {
6413 if ( pVM->hm.s.fAllow64BitGuests
6414 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6415 {
6416 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6417 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6418 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6419 }
6420 }
6421#endif
6422
6423 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6424#if HC_ARCH_BITS == 32
6425 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6426#endif
6427 )
6428 {
6429 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6430 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6431 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6432 {
6433 switch (pMsr->u32Msr)
6434 {
6435#if HC_ARCH_BITS == 32
6436 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6437 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6438 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6439 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6440#endif
6441 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6442 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6443 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6444 default:
6445 {
6446 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6447 ASMSetFlags(fEFlags);
6448 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6449 cMsrs));
6450 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6451 }
6452 }
6453 }
6454 }
6455
6456 if (fWhat & CPUMCTX_EXTRN_DR7)
6457 {
6458 if (!pVCpu->hm.s.fUsingHyperDR7)
6459 {
6460 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6461 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6462 VMXLOCAL_BREAK_RC(rc);
6463 pCtx->dr[7] = u32Val;
6464 }
6465 }
6466
6467 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6468 {
6469 uint32_t u32Shadow;
6470 if (fWhat & CPUMCTX_EXTRN_CR0)
6471 {
6472 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6473 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6474 VMXLOCAL_BREAK_RC(rc);
6475 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6476 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6477 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6478 CPUMSetGuestCR0(pVCpu, u32Val);
6479 VMMRZCallRing3Enable(pVCpu);
6480 }
6481
6482 if (fWhat & CPUMCTX_EXTRN_CR4)
6483 {
6484 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6485 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6486 VMXLOCAL_BREAK_RC(rc);
6487 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6488 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6489 CPUMSetGuestCR4(pVCpu, u32Val);
6490 }
6491
6492 if (fWhat & CPUMCTX_EXTRN_CR3)
6493 {
6494 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6495 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6496 || ( pVM->hm.s.fNestedPaging
6497 && CPUMIsGuestPagingEnabledEx(pCtx)))
6498 {
6499 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6500 if (pCtx->cr3 != u64Val)
6501 {
6502 CPUMSetGuestCR3(pVCpu, u64Val);
6503 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6504 }
6505
6506 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6507 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6508 if (CPUMIsGuestInPAEModeEx(pCtx))
6509 {
6510 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6511 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6512 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6513 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6514 VMXLOCAL_BREAK_RC(rc);
6515 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6516 }
6517 }
6518 }
6519 }
6520 } while (0);
6521
6522 if (RT_SUCCESS(rc))
6523 {
6524 /* Update fExtrn. */
6525 pCtx->fExtrn &= ~fWhat;
6526
6527 /* If everything has been imported, clear the HM keeper bit. */
6528 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6529 {
6530 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6531 Assert(!pCtx->fExtrn);
6532 }
6533 }
6534 }
6535 else
6536 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6537
6538 ASMSetFlags(fEFlags);
6539
6540 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6541
6542 /*
6543 * Honor any pending CR3 updates.
6544 *
6545 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6546 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6547 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6548 *
6549 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6550 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6551 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6552 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6553 *
6554 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6555 */
6556 if (VMMRZCallRing3IsEnabled(pVCpu))
6557 {
6558 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6559 {
6560 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6561 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6562 }
6563
6564 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6565 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6566
6567 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6568 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6569 }
6570
6571 return VINF_SUCCESS;
6572#undef VMXLOCAL_BREAK_RC
6573}
6574
6575
6576/**
6577 * Saves the guest state from the VMCS into the guest-CPU context.
6578 *
6579 * @returns VBox status code.
6580 * @param pVCpu The cross context virtual CPU structure.
6581 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6582 */
6583VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6584{
6585 return hmR0VmxImportGuestState(pVCpu, fWhat);
6586}
6587
6588
6589/**
6590 * Check per-VM and per-VCPU force flag actions that require us to go back to
6591 * ring-3 for one reason or another.
6592 *
6593 * @returns Strict VBox status code (i.e. informational status codes too)
6594 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6595 * ring-3.
6596 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6597 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6598 * interrupts)
6599 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6600 * all EMTs to be in ring-3.
6601 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6602 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6603 * to the EM loop.
6604 *
6605 * @param pVCpu The cross context virtual CPU structure.
6606 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6607 */
6608static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, bool fStepping)
6609{
6610 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6611
6612 /*
6613 * Anything pending? Should be more likely than not if we're doing a good job.
6614 */
6615 PVM pVM = pVCpu->CTX_SUFF(pVM);
6616 if ( !fStepping
6617 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6618 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6619 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6620 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6621 return VINF_SUCCESS;
6622
6623 /* Pending PGM C3 sync. */
6624 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6625 {
6626 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6627 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6628 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pCtx->cr0, pCtx->cr3, pCtx->cr4,
6629 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6630 if (rcStrict2 != VINF_SUCCESS)
6631 {
6632 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6633 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6634 return rcStrict2;
6635 }
6636 }
6637
6638 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6639 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6640 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6641 {
6642 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6643 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6644 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6645 return rc2;
6646 }
6647
6648 /* Pending VM request packets, such as hardware interrupts. */
6649 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6650 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6651 {
6652 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6653 return VINF_EM_PENDING_REQUEST;
6654 }
6655
6656 /* Pending PGM pool flushes. */
6657 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6658 {
6659 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6660 return VINF_PGM_POOL_FLUSH_PENDING;
6661 }
6662
6663 /* Pending DMA requests. */
6664 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6665 {
6666 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6667 return VINF_EM_RAW_TO_R3;
6668 }
6669
6670 return VINF_SUCCESS;
6671}
6672
6673
6674/**
6675 * Converts any TRPM trap into a pending HM event. This is typically used when
6676 * entering from ring-3 (not longjmp returns).
6677 *
6678 * @param pVCpu The cross context virtual CPU structure.
6679 */
6680static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6681{
6682 Assert(TRPMHasTrap(pVCpu));
6683 Assert(!pVCpu->hm.s.Event.fPending);
6684
6685 uint8_t uVector;
6686 TRPMEVENT enmTrpmEvent;
6687 RTGCUINT uErrCode;
6688 RTGCUINTPTR GCPtrFaultAddress;
6689 uint8_t cbInstr;
6690
6691 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6692 AssertRC(rc);
6693
6694 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6695 uint32_t u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID;
6696 if (enmTrpmEvent == TRPM_TRAP)
6697 {
6698 switch (uVector)
6699 {
6700 case X86_XCPT_NMI:
6701 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6702 break;
6703
6704 case X86_XCPT_BP:
6705 case X86_XCPT_OF:
6706 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6707 break;
6708
6709 case X86_XCPT_PF:
6710 case X86_XCPT_DF:
6711 case X86_XCPT_TS:
6712 case X86_XCPT_NP:
6713 case X86_XCPT_SS:
6714 case X86_XCPT_GP:
6715 case X86_XCPT_AC:
6716 u32IntInfo |= VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
6717 RT_FALL_THRU();
6718 default:
6719 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6720 break;
6721 }
6722 }
6723 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6724 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6725 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6726 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_SW_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
6727 else
6728 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6729
6730 rc = TRPMResetTrap(pVCpu);
6731 AssertRC(rc);
6732 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6733 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6734
6735 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6736}
6737
6738
6739/**
6740 * Converts the pending HM event into a TRPM trap.
6741 *
6742 * @param pVCpu The cross context virtual CPU structure.
6743 */
6744static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6745{
6746 Assert(pVCpu->hm.s.Event.fPending);
6747
6748 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6749 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6750 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVCpu->hm.s.Event.u64IntInfo);
6751 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6752
6753 /* If a trap was already pending, we did something wrong! */
6754 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6755
6756 TRPMEVENT enmTrapType;
6757 switch (uVectorType)
6758 {
6759 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6760 enmTrapType = TRPM_HARDWARE_INT;
6761 break;
6762
6763 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6764 enmTrapType = TRPM_SOFTWARE_INT;
6765 break;
6766
6767 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6768 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6769 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6770 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6771 enmTrapType = TRPM_TRAP;
6772 break;
6773
6774 default:
6775 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6776 enmTrapType = TRPM_32BIT_HACK;
6777 break;
6778 }
6779
6780 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6781
6782 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6783 AssertRC(rc);
6784
6785 if (fErrorCodeValid)
6786 TRPMSetErrorCode(pVCpu, uErrorCode);
6787
6788 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6789 && uVector == X86_XCPT_PF)
6790 {
6791 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6792 }
6793 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6794 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6795 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6796 {
6797 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6798 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6799 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6800 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6801 }
6802
6803 /* Clear any pending events from the VMCS. */
6804 VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0);
6805 VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, 0);
6806
6807 /* We're now done converting the pending event. */
6808 pVCpu->hm.s.Event.fPending = false;
6809}
6810
6811
6812/**
6813 * Does the necessary state syncing before returning to ring-3 for any reason
6814 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6815 *
6816 * @returns VBox status code.
6817 * @param pVCpu The cross context virtual CPU structure.
6818 * @param fImportState Whether to import the guest state from the VMCS back
6819 * to the guest-CPU context.
6820 *
6821 * @remarks No-long-jmp zone!!!
6822 */
6823static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6824{
6825 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6826 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6827
6828 RTCPUID idCpu = RTMpCpuId();
6829 Log4Func(("HostCpuId=%u\n", idCpu));
6830
6831 /*
6832 * !!! IMPORTANT !!!
6833 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6834 */
6835
6836 /* Save the guest state if necessary. */
6837 if (fImportState)
6838 {
6839 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6840 AssertRCReturn(rc, rc);
6841 }
6842
6843 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6844 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6845 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6846
6847 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6848#ifdef VBOX_STRICT
6849 if (CPUMIsHyperDebugStateActive(pVCpu))
6850 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MOV_DR_EXIT);
6851#endif
6852 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6853 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6854 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6855
6856#if HC_ARCH_BITS == 64
6857 /* Restore host-state bits that VT-x only restores partially. */
6858 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6859 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6860 {
6861 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6862 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6863 }
6864 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6865#endif
6866
6867 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6868 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6869 {
6870 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6871 if (!fImportState)
6872 {
6873 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS);
6874 AssertRCReturn(rc, rc);
6875 }
6876 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6877 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6878 }
6879 else
6880 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6881
6882 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6883 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6884
6885 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6886 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6887 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6888 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6889 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6890 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6891 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6892 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6893 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6894
6895 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6896
6897 /** @todo This partially defeats the purpose of having preemption hooks.
6898 * The problem is, deregistering the hooks should be moved to a place that
6899 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6900 * context.
6901 */
6902 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6903 {
6904 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6905 AssertRCReturn(rc, rc);
6906
6907 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6908 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6909 }
6910 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6911 NOREF(idCpu);
6912
6913 return VINF_SUCCESS;
6914}
6915
6916
6917/**
6918 * Leaves the VT-x session.
6919 *
6920 * @returns VBox status code.
6921 * @param pVCpu The cross context virtual CPU structure.
6922 *
6923 * @remarks No-long-jmp zone!!!
6924 */
6925static int hmR0VmxLeaveSession(PVMCPU pVCpu)
6926{
6927 HM_DISABLE_PREEMPT(pVCpu);
6928 HMVMX_ASSERT_CPU_SAFE(pVCpu);
6929 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6930 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6931
6932 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6933 and done this from the VMXR0ThreadCtxCallback(). */
6934 if (!pVCpu->hm.s.fLeaveDone)
6935 {
6936 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6937 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6938 pVCpu->hm.s.fLeaveDone = true;
6939 }
6940 Assert(!pVCpu->cpum.GstCtx.fExtrn);
6941
6942 /*
6943 * !!! IMPORTANT !!!
6944 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6945 */
6946
6947 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6948 /** @todo Deregistering here means we need to VMCLEAR always
6949 * (longjmp/exit-to-r3) in VT-x which is not efficient, eliminate need
6950 * for calling VMMR0ThreadCtxHookDisable here! */
6951 VMMR0ThreadCtxHookDisable(pVCpu);
6952
6953 /* Leave HM context. This takes care of local init (term). */
6954 int rc = HMR0LeaveCpu(pVCpu);
6955
6956 HM_RESTORE_PREEMPT();
6957 return rc;
6958}
6959
6960
6961/**
6962 * Does the necessary state syncing before doing a longjmp to ring-3.
6963 *
6964 * @returns VBox status code.
6965 * @param pVCpu The cross context virtual CPU structure.
6966 *
6967 * @remarks No-long-jmp zone!!!
6968 */
6969DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu)
6970{
6971 return hmR0VmxLeaveSession(pVCpu);
6972}
6973
6974
6975/**
6976 * Take necessary actions before going back to ring-3.
6977 *
6978 * An action requires us to go back to ring-3. This function does the necessary
6979 * steps before we can safely return to ring-3. This is not the same as longjmps
6980 * to ring-3, this is voluntary and prepares the guest so it may continue
6981 * executing outside HM (recompiler/IEM).
6982 *
6983 * @returns VBox status code.
6984 * @param pVCpu The cross context virtual CPU structure.
6985 * @param rcExit The reason for exiting to ring-3. Can be
6986 * VINF_VMM_UNKNOWN_RING3_CALL.
6987 */
6988static int hmR0VmxExitToRing3(PVMCPU pVCpu, VBOXSTRICTRC rcExit)
6989{
6990 Assert(pVCpu);
6991 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
6992
6993 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6994 {
6995 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6996 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6997 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6998 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6999 }
7000
7001 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7002 VMMRZCallRing3Disable(pVCpu);
7003 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7004
7005 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7006 if (pVCpu->hm.s.Event.fPending)
7007 {
7008 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7009 Assert(!pVCpu->hm.s.Event.fPending);
7010 }
7011
7012 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7013 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7014
7015 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7016 and if we're injecting an event we should have a TRPM trap pending. */
7017 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7018#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7019 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7020#endif
7021
7022 /* Save guest state and restore host state bits. */
7023 int rc = hmR0VmxLeaveSession(pVCpu);
7024 AssertRCReturn(rc, rc);
7025 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7026 /* Thread-context hooks are unregistered at this point!!! */
7027
7028 /* Sync recompiler state. */
7029 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7030 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7031 | CPUM_CHANGED_LDTR
7032 | CPUM_CHANGED_GDTR
7033 | CPUM_CHANGED_IDTR
7034 | CPUM_CHANGED_TR
7035 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7036 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7037 && CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx))
7038 {
7039 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7040 }
7041
7042 Assert(!pVCpu->hm.s.fClearTrapFlag);
7043
7044 /* Update the exit-to-ring 3 reason. */
7045 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7046
7047 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7048 if (rcExit != VINF_EM_RAW_INTERRUPT)
7049 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7050
7051 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7052
7053 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7054 VMMRZCallRing3RemoveNotification(pVCpu);
7055 VMMRZCallRing3Enable(pVCpu);
7056
7057 return rc;
7058}
7059
7060
7061/**
7062 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7063 * longjump to ring-3 and possibly get preempted.
7064 *
7065 * @returns VBox status code.
7066 * @param pVCpu The cross context virtual CPU structure.
7067 * @param enmOperation The operation causing the ring-3 longjump.
7068 * @param pvUser User argument, currently unused, NULL.
7069 */
7070static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7071{
7072 RT_NOREF(pvUser);
7073 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7074 {
7075 /*
7076 * !!! IMPORTANT !!!
7077 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7078 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7079 */
7080 VMMRZCallRing3RemoveNotification(pVCpu);
7081 VMMRZCallRing3Disable(pVCpu);
7082 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7083 RTThreadPreemptDisable(&PreemptState);
7084
7085 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
7086 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7087 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7088
7089#if HC_ARCH_BITS == 64
7090 /* Restore host-state bits that VT-x only restores partially. */
7091 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7092 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7093 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7094 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7095#endif
7096
7097 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7098 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7099 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7100
7101 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7102 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7103 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7104 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7105 {
7106 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7107 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7108 }
7109
7110 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7111 VMMR0ThreadCtxHookDisable(pVCpu);
7112 HMR0LeaveCpu(pVCpu);
7113 RTThreadPreemptRestore(&PreemptState);
7114 return VINF_SUCCESS;
7115 }
7116
7117 Assert(pVCpu);
7118 Assert(pvUser);
7119 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7120 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7121
7122 VMMRZCallRing3Disable(pVCpu);
7123 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7124
7125 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7126
7127 int rc = hmR0VmxLongJmpToRing3(pVCpu);
7128 AssertRCReturn(rc, rc);
7129
7130 VMMRZCallRing3Enable(pVCpu);
7131 return VINF_SUCCESS;
7132}
7133
7134
7135/**
7136 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7137 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7138 *
7139 * @param pVCpu The cross context virtual CPU structure.
7140 */
7141DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7142{
7143 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7144 {
7145 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT))
7146 {
7147 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_INT_WINDOW_EXIT;
7148 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7149 AssertRC(rc);
7150 Log4Func(("Setup interrupt-window exiting\n"));
7151 }
7152 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7153}
7154
7155
7156/**
7157 * Clears the interrupt-window exiting control in the VMCS.
7158 *
7159 * @param pVCpu The cross context virtual CPU structure.
7160 */
7161DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7162{
7163 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT);
7164 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_INT_WINDOW_EXIT;
7165 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7166 AssertRC(rc);
7167 Log4Func(("Cleared interrupt-window exiting\n"));
7168}
7169
7170
7171/**
7172 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7173 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7174 *
7175 * @param pVCpu The cross context virtual CPU structure.
7176 */
7177DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7178{
7179 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7180 {
7181 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT))
7182 {
7183 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7184 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7185 AssertRC(rc);
7186 Log4Func(("Setup NMI-window exiting\n"));
7187 }
7188 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7189}
7190
7191
7192/**
7193 * Clears the NMI-window exiting control in the VMCS.
7194 *
7195 * @param pVCpu The cross context virtual CPU structure.
7196 */
7197DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7198{
7199 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT);
7200 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_NMI_WINDOW_EXIT;
7201 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7202 AssertRC(rc);
7203 Log4Func(("Cleared NMI-window exiting\n"));
7204}
7205
7206
7207/**
7208 * Evaluates the event to be delivered to the guest and sets it as the pending
7209 * event.
7210 *
7211 * @returns The VT-x guest-interruptibility state.
7212 * @param pVCpu The cross context virtual CPU structure.
7213 */
7214static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu)
7215{
7216 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7217 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7218 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu);
7219 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7220 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7221 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7222
7223 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7224 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7225 Assert(!fBlockSti || pCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7226 Assert(!TRPMHasTrap(pVCpu));
7227
7228 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7229 APICUpdatePendingInterrupts(pVCpu);
7230
7231 /*
7232 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7233 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7234 */
7235 /** @todo SMI. SMIs take priority over NMIs. */
7236 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7237 {
7238 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7239 if ( !pVCpu->hm.s.Event.fPending
7240 && !fBlockNmi
7241 && !fBlockSti
7242 && !fBlockMovSS)
7243 {
7244 Log4Func(("Pending NMI\n"));
7245 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INT_INFO_VALID;
7246 u32IntInfo |= (VMX_EXIT_INT_INFO_TYPE_NMI << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7247
7248 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7249 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7250 }
7251 else
7252 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7253 }
7254 /*
7255 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7256 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7257 */
7258 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7259 && !pVCpu->hm.s.fSingleInstruction)
7260 {
7261 Assert(!DBGFIsStepping(pVCpu));
7262 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7263 AssertRCReturn(rc, 0);
7264 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7265 if ( !pVCpu->hm.s.Event.fPending
7266 && !fBlockInt
7267 && !fBlockSti
7268 && !fBlockMovSS)
7269 {
7270 uint8_t u8Interrupt;
7271 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7272 if (RT_SUCCESS(rc))
7273 {
7274 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7275 uint32_t u32IntInfo = u8Interrupt
7276 | VMX_EXIT_INT_INFO_VALID
7277 | (VMX_EXIT_INT_INFO_TYPE_EXT_INT << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7278
7279 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7280 }
7281 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7282 {
7283 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
7284 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7285 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7286
7287 /*
7288 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7289 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7290 * need to re-set this force-flag here.
7291 */
7292 }
7293 else
7294 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7295 }
7296 else
7297 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7298 }
7299
7300 return fIntrState;
7301}
7302
7303
7304/**
7305 * Sets a pending-debug exception to be delivered to the guest if the guest is
7306 * single-stepping in the VMCS.
7307 *
7308 * @param pVCpu The cross context virtual CPU structure.
7309 */
7310DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu)
7311{
7312 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7313 RT_NOREF(pVCpu);
7314 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, VMX_VMCS_GUEST_PENDING_DEBUG_XCPT_BS);
7315}
7316
7317
7318/**
7319 * Injects any pending events into the guest if the guest is in a state to
7320 * receive them.
7321 *
7322 * @returns Strict VBox status code (i.e. informational status codes too).
7323 * @param pVCpu The cross context virtual CPU structure.
7324 * @param fIntrState The VT-x guest-interruptibility state.
7325 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7326 * return VINF_EM_DBG_STEPPED if the event was
7327 * dispatched directly.
7328 */
7329static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, uint32_t fIntrState, bool fStepping)
7330{
7331 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7332 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7333
7334 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7335 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7336
7337 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7338 Assert(!(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7339 Assert(!fBlockSti || pVCpu->cpum.GstCtx.eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7340 Assert(!TRPMHasTrap(pVCpu));
7341
7342 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7343 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7344 if (pVCpu->hm.s.Event.fPending)
7345 {
7346 /*
7347 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7348 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7349 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7350 *
7351 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7352 */
7353 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7354#ifdef VBOX_STRICT
7355 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7356 {
7357 bool const fBlockInt = !(pCtx->eflags.u32 & X86_EFL_IF);
7358 Assert(!fBlockInt);
7359 Assert(!fBlockSti);
7360 Assert(!fBlockMovSS);
7361 }
7362 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
7363 {
7364 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI);
7365 Assert(!fBlockSti);
7366 Assert(!fBlockMovSS);
7367 Assert(!fBlockNmi);
7368 }
7369#endif
7370 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7371 uIntType));
7372 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7373 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7374 &fIntrState);
7375 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7376
7377 /* Update the interruptibility-state as it could have been changed by
7378 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7379 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS);
7380 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
7381
7382 if (uIntType == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
7383 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7384 else
7385 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7386 }
7387
7388 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7389 if ( fBlockSti
7390 || fBlockMovSS)
7391 {
7392 if (!pVCpu->hm.s.fSingleInstruction)
7393 {
7394 /*
7395 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7396 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7397 * See Intel spec. 27.3.4 "Saving Non-Register State".
7398 */
7399 Assert(!DBGFIsStepping(pVCpu));
7400 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7401 AssertRCReturn(rc, rc);
7402 if (pCtx->eflags.Bits.u1TF)
7403 {
7404 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
7405 AssertRCReturn(rc2, rc2);
7406 }
7407 }
7408 else if (pCtx->eflags.Bits.u1TF)
7409 {
7410 /*
7411 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7412 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7413 */
7414 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.ProcCtls.n.allowed1 & VMX_PROC_CTLS_MONITOR_TRAP_FLAG));
7415 fIntrState = 0;
7416 }
7417 }
7418
7419 /*
7420 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7421 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7422 */
7423 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7424 AssertRCReturn(rc3, rc3);
7425
7426 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7427 NOREF(fBlockMovSS); NOREF(fBlockSti);
7428 return rcStrict;
7429}
7430
7431
7432/**
7433 * Injects a double-fault (\#DF) exception into the VM.
7434 *
7435 * @returns Strict VBox status code (i.e. informational status codes too).
7436 * @param pVCpu The cross context virtual CPU structure.
7437 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7438 * and should return VINF_EM_DBG_STEPPED if the event
7439 * is injected directly (register modified by us, not
7440 * by hardware on VM-entry).
7441 * @param pfIntrState Pointer to the current guest interruptibility-state.
7442 * This interruptibility-state will be updated if
7443 * necessary. This cannot not be NULL.
7444 */
7445DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, bool fStepping, uint32_t *pfIntrState)
7446{
7447 uint32_t const u32IntInfo = X86_XCPT_DF | VMX_EXIT_INT_INFO_VALID
7448 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7449 | VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7450 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7451 pfIntrState);
7452}
7453
7454
7455/**
7456 * Injects a general-protection (\#GP) fault into the VM.
7457 *
7458 * @returns Strict VBox status code (i.e. informational status codes too).
7459 * @param pVCpu The cross context virtual CPU structure.
7460 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7461 * mode, i.e. in real-mode it's not valid).
7462 * @param u32ErrorCode The error code associated with the \#GP.
7463 * @param fStepping Whether we're running in
7464 * hmR0VmxRunGuestCodeStep() and should return
7465 * VINF_EM_DBG_STEPPED if the event is injected
7466 * directly (register modified by us, not by
7467 * hardware on VM-entry).
7468 * @param pfIntrState Pointer to the current guest interruptibility-state.
7469 * This interruptibility-state will be updated if
7470 * necessary. This cannot not be NULL.
7471 */
7472DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, bool fErrorCodeValid, uint32_t u32ErrorCode, bool fStepping,
7473 uint32_t *pfIntrState)
7474{
7475 uint32_t const u32IntInfo = X86_XCPT_GP | VMX_EXIT_INT_INFO_VALID
7476 | (VMX_EXIT_INT_INFO_TYPE_HW_XCPT << VMX_EXIT_INT_INFO_TYPE_SHIFT)
7477 | (fErrorCodeValid ? VMX_EXIT_INT_INFO_ERROR_CODE_VALID : 0);
7478 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7479 pfIntrState);
7480}
7481
7482
7483/**
7484 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7485 *
7486 * @param pVCpu The cross context virtual CPU structure.
7487 * @param uVector The software interrupt vector number.
7488 * @param cbInstr The value of RIP that is to be pushed on the guest
7489 * stack.
7490 */
7491DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, uint16_t uVector, uint32_t cbInstr)
7492{
7493 bool const fIsSwXcpt = RT_BOOL(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7494 uint32_t const u32IntType = fIsSwXcpt ? VMX_EXIT_INT_INFO_TYPE_SW_XCPT : VMX_EXIT_INT_INFO_TYPE_SW_INT;
7495 uint32_t const u32IntInfo = uVector | VMX_EXIT_INT_INFO_VALID
7496 | (u32IntType << VMX_EXIT_INT_INFO_TYPE_SHIFT);
7497 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7498}
7499
7500
7501/**
7502 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7503 * stack.
7504 *
7505 * @returns Strict VBox status code (i.e. informational status codes too).
7506 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7507 * @param pVCpu The cross context virtual CPU structure.
7508 * @param uValue The value to push to the guest stack.
7509 */
7510static VBOXSTRICTRC hmR0VmxRealModeGuestStackPush(PVMCPU pVCpu, uint16_t uValue)
7511{
7512 /*
7513 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7514 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7515 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7516 */
7517 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7518 if (pCtx->sp == 1)
7519 return VINF_EM_RESET;
7520 pCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7521 int rc = PGMPhysSimpleWriteGCPhys(pVCpu->CTX_SUFF(pVM), pCtx->ss.u64Base + pCtx->sp, &uValue, sizeof(uint16_t));
7522 AssertRC(rc);
7523 return rc;
7524}
7525
7526
7527/**
7528 * Injects an event into the guest upon VM-entry by updating the relevant fields
7529 * in the VM-entry area in the VMCS.
7530 *
7531 * @returns Strict VBox status code (i.e. informational status codes too).
7532 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7533 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7534 *
7535 * @param pVCpu The cross context virtual CPU structure.
7536 * @param u64IntInfo The VM-entry interruption-information field.
7537 * @param cbInstr The VM-entry instruction length in bytes (for
7538 * software interrupts, exceptions and privileged
7539 * software exceptions).
7540 * @param u32ErrCode The VM-entry exception error code.
7541 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7542 * @param pfIntrState Pointer to the current guest interruptibility-state.
7543 * This interruptibility-state will be updated if
7544 * necessary. This cannot not be NULL.
7545 * @param fStepping Whether we're running in
7546 * hmR0VmxRunGuestCodeStep() and should return
7547 * VINF_EM_DBG_STEPPED if the event is injected
7548 * directly (register modified by us, not by
7549 * hardware on VM-entry).
7550 */
7551static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7552 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7553{
7554 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7555 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7556 Assert(pfIntrState);
7557
7558 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
7559 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7560 uint32_t const uVector = VMX_EXIT_INT_INFO_VECTOR(u32IntInfo);
7561 uint32_t const uIntType = VMX_EXIT_INT_INFO_TYPE(u32IntInfo);
7562
7563#ifdef VBOX_STRICT
7564 /*
7565 * Validate the error-code-valid bit for hardware exceptions.
7566 * No error codes for exceptions in real-mode.
7567 *
7568 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7569 */
7570 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7571 && !CPUMIsGuestInRealModeEx(pCtx))
7572 {
7573 switch (uVector)
7574 {
7575 case X86_XCPT_PF:
7576 case X86_XCPT_DF:
7577 case X86_XCPT_TS:
7578 case X86_XCPT_NP:
7579 case X86_XCPT_SS:
7580 case X86_XCPT_GP:
7581 case X86_XCPT_AC:
7582 AssertMsg(VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo),
7583 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7584 RT_FALL_THRU();
7585 default:
7586 break;
7587 }
7588 }
7589#endif
7590
7591 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7592 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7593 || !(*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS));
7594
7595 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7596
7597 /*
7598 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7599 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7600 * interrupt handler in the (real-mode) guest.
7601 *
7602 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7603 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7604 */
7605 if (CPUMIsGuestInRealModeEx(pCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7606 {
7607 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7608 {
7609 /*
7610 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7611 * set the deliver-error-code bit.
7612 *
7613 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7614 */
7615 u32IntInfo &= ~VMX_EXIT_INT_INFO_ERROR_CODE_VALID;
7616 }
7617 else
7618 {
7619 PVM pVM = pVCpu->CTX_SUFF(pVM);
7620 Assert(PDMVmmDevHeapIsEnabled(pVM));
7621 Assert(pVM->hm.s.vmx.pRealModeTSS);
7622
7623 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7624 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_TABLE_MASK | CPUMCTX_EXTRN_RIP
7625 | CPUMCTX_EXTRN_RSP | CPUMCTX_EXTRN_RFLAGS);
7626 AssertRCReturn(rc2, rc2);
7627
7628 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7629 size_t const cbIdtEntry = sizeof(X86IDTR16);
7630 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pCtx->idtr.cbIdt)
7631 {
7632 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7633 if (uVector == X86_XCPT_DF)
7634 return VINF_EM_RESET;
7635
7636 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7637 if (uVector == X86_XCPT_GP)
7638 return hmR0VmxInjectXcptDF(pVCpu, fStepping, pfIntrState);
7639
7640 /*
7641 * If we're injecting an event with no valid IDT entry, inject a #GP.
7642 * No error codes for exceptions in real-mode.
7643 *
7644 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7645 */
7646 return hmR0VmxInjectXcptGP(pVCpu, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping, pfIntrState);
7647 }
7648
7649 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7650 uint16_t uGuestIp = pCtx->ip;
7651 if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_XCPT)
7652 {
7653 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7654 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7655 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7656 }
7657 else if (uIntType == VMX_EXIT_INT_INFO_TYPE_SW_INT)
7658 uGuestIp = pCtx->ip + (uint16_t)cbInstr;
7659
7660 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7661 X86IDTR16 IdtEntry;
7662 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pCtx->idtr.pIdt + uVector * cbIdtEntry;
7663 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7664 AssertRCReturn(rc2, rc2);
7665
7666 /* Construct the stack frame for the interrupt/exception handler. */
7667 VBOXSTRICTRC rcStrict;
7668 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->eflags.u32);
7669 if (rcStrict == VINF_SUCCESS)
7670 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, pCtx->cs.Sel);
7671 if (rcStrict == VINF_SUCCESS)
7672 rcStrict = hmR0VmxRealModeGuestStackPush(pVCpu, uGuestIp);
7673
7674 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7675 if (rcStrict == VINF_SUCCESS)
7676 {
7677 pCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7678 pCtx->rip = IdtEntry.offSel;
7679 pCtx->cs.Sel = IdtEntry.uSel;
7680 pCtx->cs.ValidSel = IdtEntry.uSel;
7681 pCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7682 if ( uIntType == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7683 && uVector == X86_XCPT_PF)
7684 pCtx->cr2 = GCPtrFaultAddress;
7685
7686 /* If any other guest-state bits are changed here, make sure to update
7687 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7688 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS | HM_CHANGED_GUEST_CR2
7689 | HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
7690 | HM_CHANGED_GUEST_RSP);
7691
7692 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7693 if (*pfIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
7694 {
7695 Assert( uIntType != VMX_EXIT_INT_INFO_TYPE_NMI
7696 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
7697 Log4Func(("Clearing inhibition due to STI\n"));
7698 *pfIntrState &= ~VMX_VMCS_GUEST_INT_STATE_BLOCK_STI;
7699 }
7700 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7701 u32IntInfo, u32ErrCode, cbInstr, pCtx->eflags.u, pCtx->cs.Sel, pCtx->eip));
7702
7703 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7704 it, if we are returning to ring-3 before executing guest code. */
7705 pVCpu->hm.s.Event.fPending = false;
7706
7707 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7708 if (fStepping)
7709 rcStrict = VINF_EM_DBG_STEPPED;
7710 }
7711 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7712 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7713 return rcStrict;
7714 }
7715 }
7716
7717 /* Validate. */
7718 Assert(VMX_EXIT_INT_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7719 Assert(!VMX_EXIT_INT_INFO_IS_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7720 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7721
7722 /* Inject. */
7723 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7724 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(u32IntInfo))
7725 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7726 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7727 AssertRCReturn(rc, rc);
7728
7729 /* Update CR2. */
7730 if ( VMX_EXIT_INT_INFO_TYPE(u32IntInfo) == VMX_EXIT_INT_INFO_TYPE_HW_XCPT
7731 && uVector == X86_XCPT_PF)
7732 pCtx->cr2 = GCPtrFaultAddress;
7733
7734 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pCtx->cr2));
7735
7736 return VINF_SUCCESS;
7737}
7738
7739
7740/**
7741 * Clears the interrupt-window exiting control in the VMCS and if necessary
7742 * clears the current event in the VMCS as well.
7743 *
7744 * @returns VBox status code.
7745 * @param pVCpu The cross context virtual CPU structure.
7746 *
7747 * @remarks Use this function only to clear events that have not yet been
7748 * delivered to the guest but are injected in the VMCS!
7749 * @remarks No-long-jump zone!!!
7750 */
7751static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7752{
7753 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_INT_WINDOW_EXIT)
7754 {
7755 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7756 Log4Func(("Cleared interrupt widow\n"));
7757 }
7758
7759 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)
7760 {
7761 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7762 Log4Func(("Cleared interrupt widow\n"));
7763 }
7764}
7765
7766
7767/**
7768 * Enters the VT-x session.
7769 *
7770 * @returns VBox status code.
7771 * @param pVCpu The cross context virtual CPU structure.
7772 * @param pHostCpu Pointer to the global CPU info struct.
7773 */
7774VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7775{
7776 AssertPtr(pVCpu);
7777 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7778 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7779 RT_NOREF(pHostCpu);
7780
7781 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7782 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7783 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7784
7785#ifdef VBOX_STRICT
7786 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7787 RTCCUINTREG uHostCR4 = ASMGetCR4();
7788 if (!(uHostCR4 & X86_CR4_VMXE))
7789 {
7790 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7791 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7792 }
7793#endif
7794
7795 /*
7796 * Load the VCPU's VMCS as the current (and active) one.
7797 */
7798 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7799 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7800 if (RT_FAILURE(rc))
7801 return rc;
7802
7803 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7804 pVCpu->hm.s.fLeaveDone = false;
7805 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7806
7807 return VINF_SUCCESS;
7808}
7809
7810
7811/**
7812 * The thread-context callback (only on platforms which support it).
7813 *
7814 * @param enmEvent The thread-context event.
7815 * @param pVCpu The cross context virtual CPU structure.
7816 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7817 * @thread EMT(pVCpu)
7818 */
7819VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7820{
7821 NOREF(fGlobalInit);
7822
7823 switch (enmEvent)
7824 {
7825 case RTTHREADCTXEVENT_OUT:
7826 {
7827 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7828 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7829 VMCPU_ASSERT_EMT(pVCpu);
7830
7831 /* No longjmps (logger flushes, locks) in this fragile context. */
7832 VMMRZCallRing3Disable(pVCpu);
7833 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7834
7835 /*
7836 * Restore host-state (FPU, debug etc.)
7837 */
7838 if (!pVCpu->hm.s.fLeaveDone)
7839 {
7840 /*
7841 * Do -not- import the guest-state here as we might already be in the middle of importing
7842 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7843 */
7844 hmR0VmxLeave(pVCpu, false /* fImportState */);
7845 pVCpu->hm.s.fLeaveDone = true;
7846 }
7847
7848 /* Leave HM context, takes care of local init (term). */
7849 int rc = HMR0LeaveCpu(pVCpu);
7850 AssertRC(rc); NOREF(rc);
7851
7852 /* Restore longjmp state. */
7853 VMMRZCallRing3Enable(pVCpu);
7854 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7855 break;
7856 }
7857
7858 case RTTHREADCTXEVENT_IN:
7859 {
7860 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7861 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7862 VMCPU_ASSERT_EMT(pVCpu);
7863
7864 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7865 VMMRZCallRing3Disable(pVCpu);
7866 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7867
7868 /* Initialize the bare minimum state required for HM. This takes care of
7869 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7870 int rc = hmR0EnterCpu(pVCpu);
7871 AssertRC(rc);
7872 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7873 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7874
7875 /* Load the active VMCS as the current one. */
7876 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7877 {
7878 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7879 AssertRC(rc); NOREF(rc);
7880 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7881 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7882 }
7883 pVCpu->hm.s.fLeaveDone = false;
7884
7885 /* Restore longjmp state. */
7886 VMMRZCallRing3Enable(pVCpu);
7887 break;
7888 }
7889
7890 default:
7891 break;
7892 }
7893}
7894
7895
7896/**
7897 * Exports the host state into the VMCS host-state area.
7898 * Sets up the VM-exit MSR-load area.
7899 *
7900 * The CPU state will be loaded from these fields on every successful VM-exit.
7901 *
7902 * @returns VBox status code.
7903 * @param pVCpu The cross context virtual CPU structure.
7904 *
7905 * @remarks No-long-jump zone!!!
7906 */
7907static int hmR0VmxExportHostState(PVMCPU pVCpu)
7908{
7909 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7910
7911 int rc = VINF_SUCCESS;
7912 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7913 {
7914 rc = hmR0VmxExportHostControlRegs();
7915 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7916
7917 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7918 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7919
7920 rc = hmR0VmxExportHostMsrs(pVCpu);
7921 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7922
7923 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7924 }
7925 return rc;
7926}
7927
7928
7929/**
7930 * Saves the host state in the VMCS host-state.
7931 *
7932 * @returns VBox status code.
7933 * @param pVCpu The cross context virtual CPU structure.
7934 *
7935 * @remarks No-long-jump zone!!!
7936 */
7937VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7938{
7939 AssertPtr(pVCpu);
7940 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7941
7942 /*
7943 * Export the host state here while entering HM context.
7944 * When thread-context hooks are used, we might get preempted and have to re-save the host
7945 * state but most of the time we won't be, so do it here before we disable interrupts.
7946 */
7947 return hmR0VmxExportHostState(pVCpu);
7948}
7949
7950
7951/**
7952 * Exports the guest state into the VMCS guest-state area.
7953 *
7954 * The will typically be done before VM-entry when the guest-CPU state and the
7955 * VMCS state may potentially be out of sync.
7956 *
7957 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7958 * VM-entry controls.
7959 * Sets up the appropriate VMX non-root function to execute guest code based on
7960 * the guest CPU mode.
7961 *
7962 * @returns VBox strict status code.
7963 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
7964 * without unrestricted guest access and the VMMDev is not presently
7965 * mapped (e.g. EFI32).
7966 *
7967 * @param pVCpu The cross context virtual CPU structure.
7968 *
7969 * @remarks No-long-jump zone!!!
7970 */
7971static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu)
7972{
7973 AssertPtr(pVCpu);
7974 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
7975
7976 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7977
7978 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
7979
7980 /* Determine real-on-v86 mode. */
7981 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
7982 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
7983 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx))
7984 {
7985 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
7986 }
7987
7988 /*
7989 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
7990 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
7991 */
7992 int rc = hmR0VmxSelectVMRunHandler(pVCpu);
7993 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7994
7995 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
7996 rc = hmR0VmxExportGuestEntryCtls(pVCpu);
7997 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7998
7999 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8000 rc = hmR0VmxExportGuestExitCtls(pVCpu);
8001 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8002
8003 rc = hmR0VmxExportGuestCR0(pVCpu);
8004 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8005
8006 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu);
8007 if (rcStrict == VINF_SUCCESS)
8008 { /* likely */ }
8009 else
8010 {
8011 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8012 return rcStrict;
8013 }
8014
8015 rc = hmR0VmxExportGuestSegmentRegs(pVCpu);
8016 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8017
8018 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8019 may alter controls if we determine we don't have to swap EFER after all. */
8020 rc = hmR0VmxExportGuestMsrs(pVCpu);
8021 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8022
8023 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8024 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8025
8026 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8027 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8028
8029 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8030 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8031 rc = hmR0VmxExportGuestRip(pVCpu);
8032 rc |= hmR0VmxExportGuestRsp(pVCpu);
8033 rc |= hmR0VmxExportGuestRflags(pVCpu);
8034 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8035
8036 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8037 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8038 | HM_CHANGED_GUEST_CR2
8039 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8040 | HM_CHANGED_GUEST_X87
8041 | HM_CHANGED_GUEST_SSE_AVX
8042 | HM_CHANGED_GUEST_OTHER_XSAVE
8043 | HM_CHANGED_GUEST_XCRx
8044 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8045 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8046 | HM_CHANGED_GUEST_TSC_AUX
8047 | HM_CHANGED_GUEST_OTHER_MSRS
8048 | HM_CHANGED_GUEST_HWVIRT
8049 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8050
8051 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8052 return rc;
8053}
8054
8055
8056/**
8057 * Exports the state shared between the host and guest into the VMCS.
8058 *
8059 * @param pVCpu The cross context virtual CPU structure.
8060 *
8061 * @remarks No-long-jump zone!!!
8062 */
8063static void hmR0VmxExportSharedState(PVMCPU pVCpu)
8064{
8065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8066 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8067
8068 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8069 {
8070 int rc = hmR0VmxExportSharedDebugState(pVCpu);
8071 AssertRC(rc);
8072 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8073
8074 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8075 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8076 {
8077 rc = hmR0VmxExportGuestRflags(pVCpu);
8078 AssertRC(rc);
8079 }
8080 }
8081
8082 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8083 {
8084 hmR0VmxLazyLoadGuestMsrs(pVCpu);
8085 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8086 }
8087
8088 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8089 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8090}
8091
8092
8093/**
8094 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8095 *
8096 * @returns Strict VBox status code (i.e. informational status codes too).
8097 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8098 * without unrestricted guest access and the VMMDev is not presently
8099 * mapped (e.g. EFI32).
8100 *
8101 * @param pVCpu The cross context virtual CPU structure.
8102 *
8103 * @remarks No-long-jump zone!!!
8104 */
8105static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu)
8106{
8107 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
8108 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8109 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8110
8111#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8113#endif
8114
8115 /*
8116 * For many exits it's only RIP that changes and hence try to export it first
8117 * without going through a lot of change flag checks.
8118 */
8119 VBOXSTRICTRC rcStrict;
8120 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8121 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8122 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8123 {
8124 rcStrict = hmR0VmxExportGuestRip(pVCpu);
8125 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8126 { /* likely */}
8127 else
8128 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8129 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8130 }
8131 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8132 {
8133 rcStrict = hmR0VmxExportGuestState(pVCpu);
8134 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8135 { /* likely */}
8136 else
8137 {
8138 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8139 VBOXSTRICTRC_VAL(rcStrict)));
8140 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8141 return rcStrict;
8142 }
8143 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8144 }
8145 else
8146 rcStrict = VINF_SUCCESS;
8147
8148#ifdef VBOX_STRICT
8149 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8150 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8151 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8152 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8153 ("fCtxChanged=%#RX64\n", fCtxChanged));
8154#endif
8155 return rcStrict;
8156}
8157
8158
8159/**
8160 * Does the preparations before executing guest code in VT-x.
8161 *
8162 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8163 * recompiler/IEM. We must be cautious what we do here regarding committing
8164 * guest-state information into the VMCS assuming we assuredly execute the
8165 * guest in VT-x mode.
8166 *
8167 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8168 * the common-state (TRPM/forceflags), we must undo those changes so that the
8169 * recompiler/IEM can (and should) use them when it resumes guest execution.
8170 * Otherwise such operations must be done when we can no longer exit to ring-3.
8171 *
8172 * @returns Strict VBox status code (i.e. informational status codes too).
8173 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8174 * have been disabled.
8175 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8176 * double-fault into the guest.
8177 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8178 * dispatched directly.
8179 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8180 *
8181 * @param pVCpu The cross context virtual CPU structure.
8182 * @param pVmxTransient Pointer to the VMX transient structure.
8183 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8184 * us ignore some of the reasons for returning to
8185 * ring-3, and return VINF_EM_DBG_STEPPED if event
8186 * dispatching took place.
8187 */
8188static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, bool fStepping)
8189{
8190 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8191
8192#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8193 PGMRZDynMapFlushAutoSet(pVCpu);
8194#endif
8195
8196 /* Check force flag actions that might require us to go back to ring-3. */
8197 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, fStepping);
8198 if (rcStrict == VINF_SUCCESS)
8199 { /* FFs doesn't get set all the time. */ }
8200 else
8201 return rcStrict;
8202
8203 /*
8204 * Setup the virtualized-APIC accesses.
8205 *
8206 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8207 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8208 *
8209 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8210 */
8211 PVM pVM = pVCpu->CTX_SUFF(pVM);
8212 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8213 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_VIRT_APIC_ACCESS)
8214 && PDMHasApic(pVM))
8215 {
8216 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8217 Assert(u64MsrApicBase);
8218 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8219
8220 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8221
8222 /* Unalias any existing mapping. */
8223 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8224 AssertRCReturn(rc, rc);
8225
8226 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8227 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8228 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8229 AssertRCReturn(rc, rc);
8230
8231 /* Update the per-VCPU cache of the APIC base MSR. */
8232 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8233 }
8234
8235 if (TRPMHasTrap(pVCpu))
8236 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8237 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu);
8238
8239 /*
8240 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8241 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8242 * also result in triple-faulting the VM.
8243 */
8244 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, fIntrState, fStepping);
8245 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8246 { /* likely */ }
8247 else
8248 {
8249 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8250 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8251 return rcStrict;
8252 }
8253
8254 /*
8255 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8256 * import CR3 themselves. We will need to update them here, as even as late as the above
8257 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8258 * the below force flags to be set.
8259 */
8260 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8261 {
8262 Assert(!(ASMAtomicUoReadU64(&pVCpu->cpum.GstCtx.fExtrn) & CPUMCTX_EXTRN_CR3));
8263 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8264 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8265 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8266 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8267 }
8268 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8269 {
8270 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8271 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8272 }
8273
8274 /*
8275 * No longjmps to ring-3 from this point on!!!
8276 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8277 * This also disables flushing of the R0-logger instance (if any).
8278 */
8279 VMMRZCallRing3Disable(pVCpu);
8280
8281 /*
8282 * Export the guest state bits.
8283 *
8284 * We cannot perform longjmps while loading the guest state because we do not preserve the
8285 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8286 * CPU migration.
8287 *
8288 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8289 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8290 * Hence, loading of the guest state needs to be done -after- injection of events.
8291 */
8292 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu);
8293 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8294 { /* likely */ }
8295 else
8296 {
8297 VMMRZCallRing3Enable(pVCpu);
8298 return rcStrict;
8299 }
8300
8301 /*
8302 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8303 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8304 * preemption disabled for a while. Since this is purly to aid the
8305 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8306 * disable interrupt on NT.
8307 *
8308 * We need to check for force-flags that could've possible been altered since we last
8309 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8310 * see @bugref{6398}).
8311 *
8312 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8313 * to ring-3 before executing guest code.
8314 */
8315 pVmxTransient->fEFlags = ASMIntDisableFlags();
8316
8317 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8318 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8319 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8320 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8321 {
8322 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8323 {
8324 pVCpu->hm.s.Event.fPending = false;
8325
8326 /*
8327 * We've injected any pending events. This is really the point of no return (to ring-3).
8328 *
8329 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8330 * returns from this function, so don't enable them here.
8331 */
8332 return VINF_SUCCESS;
8333 }
8334
8335 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPendingHostIrq);
8336 rcStrict = VINF_EM_RAW_INTERRUPT;
8337 }
8338 else
8339 {
8340 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8341 rcStrict = VINF_EM_RAW_TO_R3;
8342 }
8343
8344 ASMSetFlags(pVmxTransient->fEFlags);
8345 VMMRZCallRing3Enable(pVCpu);
8346
8347 return rcStrict;
8348}
8349
8350
8351/**
8352 * Prepares to run guest code in VT-x and we've committed to doing so. This
8353 * means there is no backing out to ring-3 or anywhere else at this
8354 * point.
8355 *
8356 * @param pVCpu The cross context virtual CPU structure.
8357 * @param pVmxTransient Pointer to the VMX transient structure.
8358 *
8359 * @remarks Called with preemption disabled.
8360 * @remarks No-long-jump zone!!!
8361 */
8362static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
8363{
8364 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8365 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8366 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8367
8368 /*
8369 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8370 */
8371 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8372 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8373
8374 PVM pVM = pVCpu->CTX_SUFF(pVM);
8375 if (!CPUMIsGuestFPUStateActive(pVCpu))
8376 {
8377 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8378 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8379 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8380 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8381 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8382 }
8383
8384 /*
8385 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8386 */
8387 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8388 && pVCpu->hm.s.vmx.cMsrs > 0)
8389 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8390
8391 /*
8392 * Re-save the host state bits as we may've been preempted (only happens when
8393 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8394 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8395 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8396 * See @bugref{8432}.
8397 */
8398 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8399 {
8400 int rc = hmR0VmxExportHostState(pVCpu);
8401 AssertRC(rc);
8402 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8403 }
8404 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8405
8406 /*
8407 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8408 */
8409 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8410 hmR0VmxExportSharedState(pVCpu);
8411 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8412
8413 /* Store status of the shared guest-host state at the time of VM-entry. */
8414#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8415 if (CPUMIsGuestInLongModeEx(&pVCpu->cpum.GstCtx))
8416 {
8417 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8418 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8419 }
8420 else
8421#endif
8422 {
8423 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8424 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8425 }
8426
8427 /*
8428 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8429 */
8430 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8431 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8432
8433 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8434 RTCPUID idCurrentCpu = pCpu->idCpu;
8435 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8436 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8437 {
8438 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8439 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8440 }
8441
8442 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8443 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8444 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8445 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8446
8447 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8448
8449 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8450 to start executing. */
8451
8452 /*
8453 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8454 */
8455 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDTSCP)
8456 {
8457 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8458 {
8459 bool fMsrUpdated;
8460 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8461 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8462 &fMsrUpdated);
8463 AssertRC(rc2);
8464 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8465 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8466 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8467 }
8468 else
8469 {
8470 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8471 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8472 }
8473 }
8474
8475 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8476 {
8477 bool fMsrUpdated;
8478 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8479 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8480 &fMsrUpdated);
8481 AssertRC(rc2);
8482 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8483 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8484 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8485 }
8486
8487#ifdef VBOX_STRICT
8488 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8489 hmR0VmxCheckHostEferMsr(pVCpu);
8490 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8491#endif
8492#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8493 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
8494 {
8495 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
8496 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8497 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8498 }
8499#endif
8500}
8501
8502
8503/**
8504 * Performs some essential restoration of state after running guest code in
8505 * VT-x.
8506 *
8507 * @param pVCpu The cross context virtual CPU structure.
8508 * @param pVmxTransient Pointer to the VMX transient structure.
8509 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8510 *
8511 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8512 *
8513 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8514 * unconditionally when it is safe to do so.
8515 */
8516static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8517{
8518 uint64_t const uHostTsc = ASMReadTSC();
8519 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8520
8521 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8522 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8523 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8524 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8525 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8526 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8527
8528 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_RDTSC_EXIT))
8529 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8530
8531 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8532 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8533 Assert(!ASMIntAreEnabled());
8534 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8535
8536#if HC_ARCH_BITS == 64
8537 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8538#endif
8539#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8540 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8541 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8542 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8543#else
8544 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8545#endif
8546#ifdef VBOX_STRICT
8547 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8548#endif
8549 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8550
8551 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8552 uint32_t uExitReason;
8553 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8554 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8555 AssertRC(rc);
8556 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8557 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INT_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8558
8559 if (rcVMRun == VINF_SUCCESS)
8560 {
8561 /*
8562 * Update the VM-exit history array here even if the VM-entry failed due to:
8563 * - Invalid guest state.
8564 * - MSR loading.
8565 * - Machine-check event.
8566 *
8567 * In any of the above cases we will still have a "valid" VM-exit reason
8568 * despite @a fVMEntryFailed being false.
8569 *
8570 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8571 *
8572 * Note! We don't have CS or RIP at this point. Will probably address that later
8573 * by amending the history entry added here.
8574 */
8575 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8576 UINT64_MAX, uHostTsc);
8577
8578 if (!pVmxTransient->fVMEntryFailed)
8579 {
8580 VMMRZCallRing3Enable(pVCpu);
8581
8582 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8583 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8584
8585#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8586 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8587 AssertRC(rc);
8588#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8589 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8590 AssertRC(rc);
8591#else
8592 /*
8593 * Import the guest-interruptibility state always as we need it while evaluating
8594 * injecting events on re-entry.
8595 *
8596 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8597 * checking for real-mode while exporting the state because all bits that cause
8598 * mode changes wrt CR0 are intercepted.
8599 */
8600 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8601 AssertRC(rc);
8602#endif
8603
8604 /*
8605 * Sync the TPR shadow with our APIC state.
8606 */
8607 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
8608 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8609 {
8610 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8611 AssertRC(rc);
8612 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8613 }
8614
8615 return;
8616 }
8617 }
8618 else
8619 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8620
8621 VMMRZCallRing3Enable(pVCpu);
8622}
8623
8624
8625/**
8626 * Runs the guest code using VT-x the normal way.
8627 *
8628 * @returns VBox status code.
8629 * @param pVCpu The cross context virtual CPU structure.
8630 *
8631 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8632 */
8633static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu)
8634{
8635 VMXTRANSIENT VmxTransient;
8636 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8637 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8638 uint32_t cLoops = 0;
8639
8640 for (;; cLoops++)
8641 {
8642 Assert(!HMR0SuspendPending());
8643 HMVMX_ASSERT_CPU_SAFE(pVCpu);
8644
8645 /* Preparatory work for running guest code, this may force us to return
8646 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8647 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8648 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, false /* fStepping */);
8649 if (rcStrict != VINF_SUCCESS)
8650 break;
8651
8652 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
8653 int rcRun = hmR0VmxRunGuest(pVCpu);
8654
8655 /* Restore any residual host-state and save any bits shared between host
8656 and guest into the guest-CPU state. Re-enables interrupts! */
8657 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8658
8659 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8660 if (RT_SUCCESS(rcRun))
8661 { /* very likely */ }
8662 else
8663 {
8664 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8665 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
8666 return rcRun;
8667 }
8668
8669 /* Profile the VM-exit. */
8670 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8671 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8672 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8673 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8674 HMVMX_START_EXIT_DISPATCH_PROF();
8675
8676 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
8677
8678 /* Handle the VM-exit. */
8679#ifdef HMVMX_USE_FUNCTION_TABLE
8680 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, &VmxTransient);
8681#else
8682 rcStrict = hmR0VmxHandleExit(pVCpu, &VmxTransient, VmxTransient.uExitReason);
8683#endif
8684 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8685 if (rcStrict == VINF_SUCCESS)
8686 {
8687 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8688 continue; /* likely */
8689 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8690 rcStrict = VINF_EM_RAW_INTERRUPT;
8691 }
8692 break;
8693 }
8694
8695 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8696 return rcStrict;
8697}
8698
8699
8700
8701/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8702 * probes.
8703 *
8704 * The following few functions and associated structure contains the bloat
8705 * necessary for providing detailed debug events and dtrace probes as well as
8706 * reliable host side single stepping. This works on the principle of
8707 * "subclassing" the normal execution loop and workers. We replace the loop
8708 * method completely and override selected helpers to add necessary adjustments
8709 * to their core operation.
8710 *
8711 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8712 * any performance for debug and analysis features.
8713 *
8714 * @{
8715 */
8716
8717/**
8718 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8719 * the debug run loop.
8720 */
8721typedef struct VMXRUNDBGSTATE
8722{
8723 /** The RIP we started executing at. This is for detecting that we stepped. */
8724 uint64_t uRipStart;
8725 /** The CS we started executing with. */
8726 uint16_t uCsStart;
8727
8728 /** Whether we've actually modified the 1st execution control field. */
8729 bool fModifiedProcCtls : 1;
8730 /** Whether we've actually modified the 2nd execution control field. */
8731 bool fModifiedProcCtls2 : 1;
8732 /** Whether we've actually modified the exception bitmap. */
8733 bool fModifiedXcptBitmap : 1;
8734
8735 /** We desire the modified the CR0 mask to be cleared. */
8736 bool fClearCr0Mask : 1;
8737 /** We desire the modified the CR4 mask to be cleared. */
8738 bool fClearCr4Mask : 1;
8739 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8740 uint32_t fCpe1Extra;
8741 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8742 uint32_t fCpe1Unwanted;
8743 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8744 uint32_t fCpe2Extra;
8745 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8746 uint32_t bmXcptExtra;
8747 /** The sequence number of the Dtrace provider settings the state was
8748 * configured against. */
8749 uint32_t uDtraceSettingsSeqNo;
8750 /** VM-exits to check (one bit per VM-exit). */
8751 uint32_t bmExitsToCheck[3];
8752
8753 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8754 uint32_t fProcCtlsInitial;
8755 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8756 uint32_t fProcCtls2Initial;
8757 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8758 uint32_t bmXcptInitial;
8759} VMXRUNDBGSTATE;
8760AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8761typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8762
8763
8764/**
8765 * Initializes the VMXRUNDBGSTATE structure.
8766 *
8767 * @param pVCpu The cross context virtual CPU structure of the
8768 * calling EMT.
8769 * @param pDbgState The structure to initialize.
8770 */
8771static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8772{
8773 pDbgState->uRipStart = pVCpu->cpum.GstCtx.rip;
8774 pDbgState->uCsStart = pVCpu->cpum.GstCtx.cs.Sel;
8775
8776 pDbgState->fModifiedProcCtls = false;
8777 pDbgState->fModifiedProcCtls2 = false;
8778 pDbgState->fModifiedXcptBitmap = false;
8779 pDbgState->fClearCr0Mask = false;
8780 pDbgState->fClearCr4Mask = false;
8781 pDbgState->fCpe1Extra = 0;
8782 pDbgState->fCpe1Unwanted = 0;
8783 pDbgState->fCpe2Extra = 0;
8784 pDbgState->bmXcptExtra = 0;
8785 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8786 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8787 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8788}
8789
8790
8791/**
8792 * Updates the VMSC fields with changes requested by @a pDbgState.
8793 *
8794 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8795 * immediately before executing guest code, i.e. when interrupts are disabled.
8796 * We don't check status codes here as we cannot easily assert or return in the
8797 * latter case.
8798 *
8799 * @param pVCpu The cross context virtual CPU structure.
8800 * @param pDbgState The debug state.
8801 */
8802static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8803{
8804 /*
8805 * Ensure desired flags in VMCS control fields are set.
8806 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8807 *
8808 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8809 * there should be no stale data in pCtx at this point.
8810 */
8811 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8812 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8813 {
8814 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8815 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8816 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8817 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8818 pDbgState->fModifiedProcCtls = true;
8819 }
8820
8821 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8822 {
8823 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8824 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8825 Log6Func(("VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8826 pDbgState->fModifiedProcCtls2 = true;
8827 }
8828
8829 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8830 {
8831 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8832 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8833 Log6Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8834 pDbgState->fModifiedXcptBitmap = true;
8835 }
8836
8837 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8838 {
8839 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8840 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8841 Log6Func(("VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8842 }
8843
8844 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8845 {
8846 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8847 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8848 Log6Func(("VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8849 }
8850}
8851
8852
8853/**
8854 * Restores VMCS fields that were changed by hmR0VmxPreRunGuestDebugStateApply for
8855 * re-entry next time around.
8856 *
8857 * @returns Strict VBox status code (i.e. informational status codes too).
8858 * @param pVCpu The cross context virtual CPU structure.
8859 * @param pDbgState The debug state.
8860 * @param rcStrict The return code from executing the guest using single
8861 * stepping.
8862 */
8863static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8864{
8865 /*
8866 * Restore VM-exit control settings as we may not reenter this function the
8867 * next time around.
8868 */
8869 /* We reload the initial value, trigger what we can of recalculations the
8870 next time around. From the looks of things, that's all that's required atm. */
8871 if (pDbgState->fModifiedProcCtls)
8872 {
8873 if (!(pDbgState->fProcCtlsInitial & VMX_PROC_CTLS_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8874 pDbgState->fProcCtlsInitial |= VMX_PROC_CTLS_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8875 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8876 AssertRCReturn(rc2, rc2);
8877 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8878 }
8879
8880 /* We're currently the only ones messing with this one, so just restore the
8881 cached value and reload the field. */
8882 if ( pDbgState->fModifiedProcCtls2
8883 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8884 {
8885 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8886 AssertRCReturn(rc2, rc2);
8887 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8888 }
8889
8890 /* If we've modified the exception bitmap, we restore it and trigger
8891 reloading and partial recalculation the next time around. */
8892 if (pDbgState->fModifiedXcptBitmap)
8893 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8894
8895 return rcStrict;
8896}
8897
8898
8899/**
8900 * Configures VM-exit controls for current DBGF and DTrace settings.
8901 *
8902 * This updates @a pDbgState and the VMCS execution control fields to reflect
8903 * the necessary VM-exits demanded by DBGF and DTrace.
8904 *
8905 * @param pVCpu The cross context virtual CPU structure.
8906 * @param pDbgState The debug state.
8907 * @param pVmxTransient Pointer to the VMX transient structure. May update
8908 * fUpdateTscOffsettingAndPreemptTimer.
8909 */
8910static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8911{
8912 /*
8913 * Take down the dtrace serial number so we can spot changes.
8914 */
8915 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8916 ASMCompilerBarrier();
8917
8918 /*
8919 * We'll rebuild most of the middle block of data members (holding the
8920 * current settings) as we go along here, so start by clearing it all.
8921 */
8922 pDbgState->bmXcptExtra = 0;
8923 pDbgState->fCpe1Extra = 0;
8924 pDbgState->fCpe1Unwanted = 0;
8925 pDbgState->fCpe2Extra = 0;
8926 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8927 pDbgState->bmExitsToCheck[i] = 0;
8928
8929 /*
8930 * Software interrupts (INT XXh) - no idea how to trigger these...
8931 */
8932 PVM pVM = pVCpu->CTX_SUFF(pVM);
8933 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8934 || VBOXVMM_INT_SOFTWARE_ENABLED())
8935 {
8936 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8937 }
8938
8939 /*
8940 * INT3 breakpoints - triggered by #BP exceptions.
8941 */
8942 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8943 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8944
8945 /*
8946 * Exception bitmap and XCPT events+probes.
8947 */
8948 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8949 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8950 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8951
8952 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8953 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
8954 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8955 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
8956 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
8957 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
8958 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
8959 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
8960 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
8961 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
8962 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
8963 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
8964 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
8965 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
8966 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
8967 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
8968 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
8969 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
8970
8971 if (pDbgState->bmXcptExtra)
8972 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8973
8974 /*
8975 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
8976 *
8977 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
8978 * So, when adding/changing/removing please don't forget to update it.
8979 *
8980 * Some of the macros are picking up local variables to save horizontal space,
8981 * (being able to see it in a table is the lesser evil here).
8982 */
8983#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
8984 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
8985 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
8986#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
8987 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8988 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8989 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8990 } else do { } while (0)
8991#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
8992 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
8993 { \
8994 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
8995 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
8996 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
8997 } else do { } while (0)
8998#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
8999 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9000 { \
9001 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9002 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9003 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9004 } else do { } while (0)
9005#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9006 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9007 { \
9008 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9009 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9010 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9011 } else do { } while (0)
9012
9013 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9014 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9015 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9016 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9017 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9018
9019 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9020 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9021 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9022 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9023 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_PROC_CTLS_HLT_EXIT); /* paranoia */
9024 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9025 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9026 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9027 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_PROC_CTLS_INVLPG_EXIT);
9028 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9029 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_PROC_CTLS_RDPMC_EXIT);
9030 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9031 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_PROC_CTLS_RDTSC_EXIT);
9032 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9033 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9034 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9035 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9036 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9037 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9038 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9039 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9040 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9041 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9042 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9043 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9044 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9045 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9046 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9047 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9048 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9049 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9050 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9051 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9052 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9053 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9054 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9055
9056 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9057 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9058 {
9059 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_APIC_TPR);
9060 AssertRC(rc);
9061
9062#if 0 /** @todo fix me */
9063 pDbgState->fClearCr0Mask = true;
9064 pDbgState->fClearCr4Mask = true;
9065#endif
9066 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9067 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_STORE_EXIT | VMX_PROC_CTLS_CR8_STORE_EXIT;
9068 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9069 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_CR3_LOAD_EXIT | VMX_PROC_CTLS_CR8_LOAD_EXIT;
9070 pDbgState->fCpe1Unwanted |= VMX_PROC_CTLS_USE_TPR_SHADOW; /* risky? */
9071 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9072 require clearing here and in the loop if we start using it. */
9073 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9074 }
9075 else
9076 {
9077 if (pDbgState->fClearCr0Mask)
9078 {
9079 pDbgState->fClearCr0Mask = false;
9080 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9081 }
9082 if (pDbgState->fClearCr4Mask)
9083 {
9084 pDbgState->fClearCr4Mask = false;
9085 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9086 }
9087 }
9088 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9089 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9090
9091 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9092 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9093 {
9094 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9095 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9096 }
9097 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9098 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9099
9100 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS); /* risky clearing this? */
9101 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9102 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_PROC_CTLS_USE_MSR_BITMAPS);
9103 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9104 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_PROC_CTLS_MWAIT_EXIT); /* paranoia */
9105 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9106 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_PROC_CTLS_MONITOR_EXIT); /* paranoia */
9107 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9108#if 0 /** @todo too slow, fix handler. */
9109 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_PROC_CTLS_PAUSE_EXIT);
9110#endif
9111 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9112
9113 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9114 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9115 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9116 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9117 {
9118 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9119 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9120 }
9121 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9122 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9123 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9125
9126 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9127 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9128 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9129 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9130 {
9131 pDbgState->fCpe2Extra |= VMX_PROC_CTLS2_DESC_TABLE_EXIT;
9132 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9133 }
9134 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9135 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9138
9139 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9141 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_PROC_CTLS_RDTSC_EXIT);
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9143 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9145 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_PROC_CTLS2_WBINVD_EXIT);
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9147 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9149 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_PROC_CTLS2_RDRAND_EXIT);
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9151 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_PROC_CTLS_INVLPG_EXIT);
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9153 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9155 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_PROC_CTLS2_RDSEED_EXIT);
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9157 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9158 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9159 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9161
9162#undef IS_EITHER_ENABLED
9163#undef SET_ONLY_XBM_IF_EITHER_EN
9164#undef SET_CPE1_XBM_IF_EITHER_EN
9165#undef SET_CPEU_XBM_IF_EITHER_EN
9166#undef SET_CPE2_XBM_IF_EITHER_EN
9167
9168 /*
9169 * Sanitize the control stuff.
9170 */
9171 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.ProcCtls2.n.allowed1;
9172 if (pDbgState->fCpe2Extra)
9173 pDbgState->fCpe1Extra |= VMX_PROC_CTLS_USE_SECONDARY_CTLS;
9174 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.ProcCtls.n.allowed1;
9175 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.ProcCtls.n.disallowed0;
9176 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_PROC_CTLS_RDTSC_EXIT))
9177 {
9178 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9179 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9180 }
9181
9182 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9183 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9184 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9185 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9186}
9187
9188
9189/**
9190 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9191 * appropriate.
9192 *
9193 * The caller has checked the VM-exit against the
9194 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9195 * already, so we don't have to do that either.
9196 *
9197 * @returns Strict VBox status code (i.e. informational status codes too).
9198 * @param pVCpu The cross context virtual CPU structure.
9199 * @param pVmxTransient Pointer to the VMX-transient structure.
9200 * @param uExitReason The VM-exit reason.
9201 *
9202 * @remarks The name of this function is displayed by dtrace, so keep it short
9203 * and to the point. No longer than 33 chars long, please.
9204 */
9205static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t uExitReason)
9206{
9207 /*
9208 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9209 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9210 *
9211 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9212 * does. Must add/change/remove both places. Same ordering, please.
9213 *
9214 * Added/removed events must also be reflected in the next section
9215 * where we dispatch dtrace events.
9216 */
9217 bool fDtrace1 = false;
9218 bool fDtrace2 = false;
9219 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9220 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9221 uint32_t uEventArg = 0;
9222#define SET_EXIT(a_EventSubName) \
9223 do { \
9224 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9225 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9226 } while (0)
9227#define SET_BOTH(a_EventSubName) \
9228 do { \
9229 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9230 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9231 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9232 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9233 } while (0)
9234 switch (uExitReason)
9235 {
9236 case VMX_EXIT_MTF:
9237 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9238
9239 case VMX_EXIT_XCPT_OR_NMI:
9240 {
9241 uint8_t const idxVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9242 switch (VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo))
9243 {
9244 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
9245 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT:
9246 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT:
9247 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9248 {
9249 if (VMX_EXIT_INT_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uExitIntInfo))
9250 {
9251 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9252 uEventArg = pVmxTransient->uExitIntErrorCode;
9253 }
9254 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9255 switch (enmEvent1)
9256 {
9257 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9258 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9259 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9260 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9261 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9262 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9263 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9264 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9265 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9266 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9267 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9268 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9269 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9270 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9271 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9272 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9273 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9274 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9275 default: break;
9276 }
9277 }
9278 else
9279 AssertFailed();
9280 break;
9281
9282 case VMX_EXIT_INT_INFO_TYPE_SW_INT:
9283 uEventArg = idxVector;
9284 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9285 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9286 break;
9287 }
9288 break;
9289 }
9290
9291 case VMX_EXIT_TRIPLE_FAULT:
9292 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9293 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9294 break;
9295 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9296 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9297 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9298 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9299 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9300
9301 /* Instruction specific VM-exits: */
9302 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9303 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9304 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9305 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9306 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9307 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9308 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9309 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9310 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9311 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9312 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9313 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9314 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9315 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9316 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9317 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9318 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9319 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9320 case VMX_EXIT_MOV_CRX:
9321 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9322 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9323 SET_BOTH(CRX_READ);
9324 else
9325 SET_BOTH(CRX_WRITE);
9326 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9327 break;
9328 case VMX_EXIT_MOV_DRX:
9329 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9330 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9331 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9332 SET_BOTH(DRX_READ);
9333 else
9334 SET_BOTH(DRX_WRITE);
9335 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9336 break;
9337 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9338 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9339 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9340 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9341 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9342 case VMX_EXIT_XDTR_ACCESS:
9343 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9344 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_XDTR_INSINFO_INSTR_ID))
9345 {
9346 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9347 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9348 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9349 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9350 }
9351 break;
9352
9353 case VMX_EXIT_TR_ACCESS:
9354 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9355 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_BF_YYTR_INSINFO_INSTR_ID))
9356 {
9357 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9358 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9359 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9360 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9361 }
9362 break;
9363
9364 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9365 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9366 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9367 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9368 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9369 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9370 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9371 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9372 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9373 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9374 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9375
9376 /* Events that aren't relevant at this point. */
9377 case VMX_EXIT_EXT_INT:
9378 case VMX_EXIT_INT_WINDOW:
9379 case VMX_EXIT_NMI_WINDOW:
9380 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9381 case VMX_EXIT_PREEMPT_TIMER:
9382 case VMX_EXIT_IO_INSTR:
9383 break;
9384
9385 /* Errors and unexpected events. */
9386 case VMX_EXIT_INIT_SIGNAL:
9387 case VMX_EXIT_SIPI:
9388 case VMX_EXIT_IO_SMI:
9389 case VMX_EXIT_SMI:
9390 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9391 case VMX_EXIT_ERR_MSR_LOAD:
9392 case VMX_EXIT_ERR_MACHINE_CHECK:
9393 break;
9394
9395 default:
9396 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9397 break;
9398 }
9399#undef SET_BOTH
9400#undef SET_EXIT
9401
9402 /*
9403 * Dtrace tracepoints go first. We do them here at once so we don't
9404 * have to copy the guest state saving and stuff a few dozen times.
9405 * Down side is that we've got to repeat the switch, though this time
9406 * we use enmEvent since the probes are a subset of what DBGF does.
9407 */
9408 if (fDtrace1 || fDtrace2)
9409 {
9410 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9411 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9412 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
9413 switch (enmEvent1)
9414 {
9415 /** @todo consider which extra parameters would be helpful for each probe. */
9416 case DBGFEVENT_END: break;
9417 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pCtx); break;
9418 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pCtx, pCtx->dr[6]); break;
9419 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pCtx); break;
9420 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pCtx); break;
9421 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pCtx); break;
9422 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pCtx); break;
9423 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pCtx); break;
9424 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pCtx); break;
9425 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pCtx, uEventArg); break;
9426 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pCtx, uEventArg); break;
9427 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pCtx, uEventArg); break;
9428 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pCtx, uEventArg); break;
9429 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pCtx, uEventArg, pCtx->cr2); break;
9430 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pCtx); break;
9431 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pCtx); break;
9432 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pCtx); break;
9433 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pCtx); break;
9434 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pCtx, uEventArg); break;
9435 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9436 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9437 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pCtx); break;
9438 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pCtx); break;
9439 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pCtx); break;
9440 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pCtx); break;
9441 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pCtx); break;
9442 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pCtx); break;
9443 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pCtx); break;
9444 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9445 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9446 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9447 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9448 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9449 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pCtx, pCtx->ecx,
9450 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9451 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pCtx); break;
9452 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pCtx); break;
9453 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pCtx); break;
9454 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pCtx); break;
9455 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pCtx); break;
9456 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pCtx); break;
9457 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pCtx); break;
9458 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pCtx); break;
9459 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pCtx); break;
9460 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pCtx); break;
9461 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pCtx); break;
9462 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pCtx); break;
9463 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pCtx); break;
9464 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pCtx); break;
9465 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pCtx); break;
9466 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pCtx); break;
9467 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pCtx); break;
9468 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pCtx); break;
9469 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pCtx); break;
9470 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pCtx); break;
9471 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pCtx); break;
9472 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pCtx); break;
9473 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pCtx); break;
9474 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pCtx); break;
9475 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pCtx); break;
9476 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pCtx); break;
9477 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pCtx); break;
9478 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pCtx); break;
9479 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pCtx); break;
9480 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pCtx); break;
9481 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pCtx); break;
9482 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pCtx); break;
9483 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9484 }
9485 switch (enmEvent2)
9486 {
9487 /** @todo consider which extra parameters would be helpful for each probe. */
9488 case DBGFEVENT_END: break;
9489 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pCtx); break;
9490 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pCtx, pCtx->eax, pCtx->ecx); break;
9491 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pCtx); break;
9492 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pCtx); break;
9493 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pCtx); break;
9494 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pCtx); break;
9495 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pCtx); break;
9496 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pCtx); break;
9497 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pCtx); break;
9498 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9499 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9500 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pCtx, (uint8_t)uEventArg); break;
9501 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pCtx, (uint8_t)uEventArg); break;
9502 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pCtx, pCtx->ecx); break;
9503 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pCtx, pCtx->ecx,
9504 RT_MAKE_U64(pCtx->eax, pCtx->edx)); break;
9505 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pCtx); break;
9506 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pCtx); break;
9507 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pCtx); break;
9508 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pCtx); break;
9509 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pCtx); break;
9510 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pCtx); break;
9511 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pCtx); break;
9512 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pCtx); break;
9513 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pCtx); break;
9514 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pCtx); break;
9515 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pCtx); break;
9516 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pCtx); break;
9517 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pCtx); break;
9518 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pCtx); break;
9519 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pCtx); break;
9520 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pCtx); break;
9521 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pCtx); break;
9522 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pCtx); break;
9523 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pCtx); break;
9524 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pCtx); break;
9525 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pCtx); break;
9526 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pCtx); break;
9527 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pCtx); break;
9528 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pCtx); break;
9529 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pCtx); break;
9530 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pCtx); break;
9531 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pCtx); break;
9532 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pCtx); break;
9533 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pCtx); break;
9534 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pCtx); break;
9535 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pCtx); break;
9536 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pCtx); break;
9537 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pCtx); break;
9538 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pCtx); break;
9539 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pCtx); break;
9540 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pCtx); break;
9541 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9542 }
9543 }
9544
9545 /*
9546 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9547 * the DBGF call will do a full check).
9548 *
9549 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9550 * Note! If we have to events, we prioritize the first, i.e. the instruction
9551 * one, in order to avoid event nesting.
9552 */
9553 PVM pVM = pVCpu->CTX_SUFF(pVM);
9554 if ( enmEvent1 != DBGFEVENT_END
9555 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9556 {
9557 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent1, DBGFEVENTCTX_HM, 1, uEventArg);
9558 if (rcStrict != VINF_SUCCESS)
9559 return rcStrict;
9560 }
9561 else if ( enmEvent2 != DBGFEVENT_END
9562 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9563 {
9564 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArgs(pVM, pVCpu, enmEvent2, DBGFEVENTCTX_HM, 1, uEventArg);
9565 if (rcStrict != VINF_SUCCESS)
9566 return rcStrict;
9567 }
9568
9569 return VINF_SUCCESS;
9570}
9571
9572
9573/**
9574 * Single-stepping VM-exit filtering.
9575 *
9576 * This is preprocessing the VM-exits and deciding whether we've gotten far
9577 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9578 * handling is performed.
9579 *
9580 * @returns Strict VBox status code (i.e. informational status codes too).
9581 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9582 * @param pVmxTransient Pointer to the VMX-transient structure.
9583 * @param pDbgState The debug state.
9584 */
9585DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, PVMXRUNDBGSTATE pDbgState)
9586{
9587 /*
9588 * Expensive (saves context) generic dtrace VM-exit probe.
9589 */
9590 uint32_t const uExitReason = pVmxTransient->uExitReason;
9591 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9592 { /* more likely */ }
9593 else
9594 {
9595 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9596 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9597 AssertRC(rc);
9598 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, &pVCpu->cpum.GstCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9599 }
9600
9601 /*
9602 * Check for host NMI, just to get that out of the way.
9603 */
9604 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9605 { /* normally likely */ }
9606 else
9607 {
9608 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9609 AssertRCReturn(rc2, rc2);
9610 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
9611 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
9612 return hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient);
9613 }
9614
9615 /*
9616 * Check for single stepping event if we're stepping.
9617 */
9618 if (pVCpu->hm.s.fSingleInstruction)
9619 {
9620 switch (uExitReason)
9621 {
9622 case VMX_EXIT_MTF:
9623 return hmR0VmxExitMtf(pVCpu, pVmxTransient);
9624
9625 /* Various events: */
9626 case VMX_EXIT_XCPT_OR_NMI:
9627 case VMX_EXIT_EXT_INT:
9628 case VMX_EXIT_TRIPLE_FAULT:
9629 case VMX_EXIT_INT_WINDOW:
9630 case VMX_EXIT_NMI_WINDOW:
9631 case VMX_EXIT_TASK_SWITCH:
9632 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9633 case VMX_EXIT_APIC_ACCESS:
9634 case VMX_EXIT_EPT_VIOLATION:
9635 case VMX_EXIT_EPT_MISCONFIG:
9636 case VMX_EXIT_PREEMPT_TIMER:
9637
9638 /* Instruction specific VM-exits: */
9639 case VMX_EXIT_CPUID:
9640 case VMX_EXIT_GETSEC:
9641 case VMX_EXIT_HLT:
9642 case VMX_EXIT_INVD:
9643 case VMX_EXIT_INVLPG:
9644 case VMX_EXIT_RDPMC:
9645 case VMX_EXIT_RDTSC:
9646 case VMX_EXIT_RSM:
9647 case VMX_EXIT_VMCALL:
9648 case VMX_EXIT_VMCLEAR:
9649 case VMX_EXIT_VMLAUNCH:
9650 case VMX_EXIT_VMPTRLD:
9651 case VMX_EXIT_VMPTRST:
9652 case VMX_EXIT_VMREAD:
9653 case VMX_EXIT_VMRESUME:
9654 case VMX_EXIT_VMWRITE:
9655 case VMX_EXIT_VMXOFF:
9656 case VMX_EXIT_VMXON:
9657 case VMX_EXIT_MOV_CRX:
9658 case VMX_EXIT_MOV_DRX:
9659 case VMX_EXIT_IO_INSTR:
9660 case VMX_EXIT_RDMSR:
9661 case VMX_EXIT_WRMSR:
9662 case VMX_EXIT_MWAIT:
9663 case VMX_EXIT_MONITOR:
9664 case VMX_EXIT_PAUSE:
9665 case VMX_EXIT_XDTR_ACCESS:
9666 case VMX_EXIT_TR_ACCESS:
9667 case VMX_EXIT_INVEPT:
9668 case VMX_EXIT_RDTSCP:
9669 case VMX_EXIT_INVVPID:
9670 case VMX_EXIT_WBINVD:
9671 case VMX_EXIT_XSETBV:
9672 case VMX_EXIT_RDRAND:
9673 case VMX_EXIT_INVPCID:
9674 case VMX_EXIT_VMFUNC:
9675 case VMX_EXIT_RDSEED:
9676 case VMX_EXIT_XSAVES:
9677 case VMX_EXIT_XRSTORS:
9678 {
9679 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9680 AssertRCReturn(rc, rc);
9681 if ( pVCpu->cpum.GstCtx.rip != pDbgState->uRipStart
9682 || pVCpu->cpum.GstCtx.cs.Sel != pDbgState->uCsStart)
9683 return VINF_EM_DBG_STEPPED;
9684 break;
9685 }
9686
9687 /* Errors and unexpected events: */
9688 case VMX_EXIT_INIT_SIGNAL:
9689 case VMX_EXIT_SIPI:
9690 case VMX_EXIT_IO_SMI:
9691 case VMX_EXIT_SMI:
9692 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9693 case VMX_EXIT_ERR_MSR_LOAD:
9694 case VMX_EXIT_ERR_MACHINE_CHECK:
9695 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9696 break;
9697
9698 default:
9699 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9700 break;
9701 }
9702 }
9703
9704 /*
9705 * Check for debugger event breakpoints and dtrace probes.
9706 */
9707 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9708 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9709 {
9710 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pVmxTransient, uExitReason);
9711 if (rcStrict != VINF_SUCCESS)
9712 return rcStrict;
9713 }
9714
9715 /*
9716 * Normal processing.
9717 */
9718#ifdef HMVMX_USE_FUNCTION_TABLE
9719 return g_apfnVMExitHandlers[uExitReason](pVCpu, pVmxTransient);
9720#else
9721 return hmR0VmxHandleExit(pVCpu, pVmxTransient, uExitReason);
9722#endif
9723}
9724
9725
9726/**
9727 * Single steps guest code using VT-x.
9728 *
9729 * @returns Strict VBox status code (i.e. informational status codes too).
9730 * @param pVCpu The cross context virtual CPU structure.
9731 *
9732 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9733 */
9734static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu)
9735{
9736 VMXTRANSIENT VmxTransient;
9737 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9738
9739 /* Set HMCPU indicators. */
9740 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9741 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9742 pVCpu->hm.s.fDebugWantRdTscExit = false;
9743 pVCpu->hm.s.fUsingDebugLoop = true;
9744
9745 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9746 VMXRUNDBGSTATE DbgState;
9747 hmR0VmxRunDebugStateInit(pVCpu, &DbgState);
9748 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9749
9750 /*
9751 * The loop.
9752 */
9753 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9754 for (uint32_t cLoops = 0; ; cLoops++)
9755 {
9756 Assert(!HMR0SuspendPending());
9757 HMVMX_ASSERT_CPU_SAFE(pVCpu);
9758 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9759
9760 /*
9761 * Preparatory work for running guest code, this may force us to return
9762 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9763 */
9764 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9765 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9766 rcStrict = hmR0VmxPreRunGuest(pVCpu, &VmxTransient, fStepping);
9767 if (rcStrict != VINF_SUCCESS)
9768 break;
9769
9770 hmR0VmxPreRunGuestCommitted(pVCpu, &VmxTransient);
9771 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9772
9773 /*
9774 * Now we can run the guest code.
9775 */
9776 int rcRun = hmR0VmxRunGuest(pVCpu);
9777
9778 /*
9779 * Restore any residual host-state and save any bits shared between host
9780 * and guest into the guest-CPU state. Re-enables interrupts!
9781 */
9782 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9783
9784 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9785 if (RT_SUCCESS(rcRun))
9786 { /* very likely */ }
9787 else
9788 {
9789 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9790 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, &VmxTransient);
9791 return rcRun;
9792 }
9793
9794 /* Profile the VM-exit. */
9795 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9797 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9798 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9799 HMVMX_START_EXIT_DISPATCH_PROF();
9800
9801 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, &pVCpu->cpum.GstCtx, VmxTransient.uExitReason);
9802
9803 /*
9804 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9805 */
9806 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, &VmxTransient, &DbgState);
9807 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9808 if (rcStrict != VINF_SUCCESS)
9809 break;
9810 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9811 {
9812 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9813 rcStrict = VINF_EM_RAW_INTERRUPT;
9814 break;
9815 }
9816
9817 /*
9818 * Stepping: Did the RIP change, if so, consider it a single step.
9819 * Otherwise, make sure one of the TFs gets set.
9820 */
9821 if (fStepping)
9822 {
9823 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
9824 AssertRC(rc);
9825 if ( pVCpu->cpum.GstCtx.rip != DbgState.uRipStart
9826 || pVCpu->cpum.GstCtx.cs.Sel != DbgState.uCsStart)
9827 {
9828 rcStrict = VINF_EM_DBG_STEPPED;
9829 break;
9830 }
9831 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9832 }
9833
9834 /*
9835 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9836 */
9837 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9838 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9839 }
9840
9841 /*
9842 * Clear the X86_EFL_TF if necessary.
9843 */
9844 if (pVCpu->hm.s.fClearTrapFlag)
9845 {
9846 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9847 AssertRC(rc);
9848 pVCpu->hm.s.fClearTrapFlag = false;
9849 pVCpu->cpum.GstCtx.eflags.Bits.u1TF = 0;
9850 }
9851 /** @todo there seems to be issues with the resume flag when the monitor trap
9852 * flag is pending without being used. Seen early in bios init when
9853 * accessing APIC page in protected mode. */
9854
9855 /*
9856 * Restore VM-exit control settings as we may not reenter this function the
9857 * next time around.
9858 */
9859 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9860
9861 /* Restore HMCPU indicators. */
9862 pVCpu->hm.s.fUsingDebugLoop = false;
9863 pVCpu->hm.s.fDebugWantRdTscExit = false;
9864 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9865
9866 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9867 return rcStrict;
9868}
9869
9870
9871/** @} */
9872
9873
9874/**
9875 * Checks if any expensive dtrace probes are enabled and we should go to the
9876 * debug loop.
9877 *
9878 * @returns true if we should use debug loop, false if not.
9879 */
9880static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9881{
9882 /* It's probably faster to OR the raw 32-bit counter variables together.
9883 Since the variables are in an array and the probes are next to one
9884 another (more or less), we have good locality. So, better read
9885 eight-nine cache lines ever time and only have one conditional, than
9886 128+ conditionals, right? */
9887 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9888 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9889 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9890 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9891 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9892 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9893 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9894 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9895 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9896 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9897 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9898 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9899 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9900 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9901 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9902 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9903 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9904 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9905 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9906 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9907 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9908 ) != 0
9909 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9910 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9911 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9912 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9913 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9914 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9915 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9916 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9917 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9918 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9919 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9920 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9921 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9922 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9923 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9924 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9925 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9926 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9927 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9928 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9929 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9930 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9931 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9932 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9933 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9934 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9935 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9936 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9937 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9938 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9939 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9940 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9941 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
9942 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
9943 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
9944 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
9945 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
9946 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
9947 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
9948 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
9949 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
9950 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
9951 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
9952 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
9953 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
9954 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
9955 ) != 0
9956 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
9957 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
9958 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
9959 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
9960 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
9961 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
9962 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
9963 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
9964 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
9965 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
9966 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
9967 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
9968 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
9969 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
9970 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
9971 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
9972 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
9973 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
9974 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
9975 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
9976 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
9977 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
9978 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
9979 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
9980 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
9981 | VBOXVMM_EXIT_STR_ENABLED_RAW()
9982 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
9983 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
9984 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
9985 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
9986 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
9987 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
9988 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
9989 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
9990 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
9991 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
9992 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
9993 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
9994 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
9995 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
9996 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
9997 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
9998 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
9999 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10000 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10001 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10002 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10003 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10004 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10005 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10006 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10007 ) != 0;
10008}
10009
10010
10011/**
10012 * Runs the guest code using VT-x.
10013 *
10014 * @returns Strict VBox status code (i.e. informational status codes too).
10015 * @param pVCpu The cross context virtual CPU structure.
10016 */
10017VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu)
10018{
10019 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10020 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10021 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10022 HMVMX_ASSERT_PREEMPT_SAFE(pVCpu);
10023
10024 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10025
10026 VBOXSTRICTRC rcStrict;
10027 if ( !pVCpu->hm.s.fUseDebugLoop
10028 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10029 && !DBGFIsStepping(pVCpu)
10030 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10031 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu);
10032 else
10033 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu);
10034
10035 if (rcStrict == VERR_EM_INTERPRETER)
10036 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10037 else if (rcStrict == VINF_EM_RESET)
10038 rcStrict = VINF_EM_TRIPLE_FAULT;
10039
10040 int rc2 = hmR0VmxExitToRing3(pVCpu, rcStrict);
10041 if (RT_FAILURE(rc2))
10042 {
10043 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10044 rcStrict = rc2;
10045 }
10046 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10047 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10048 return rcStrict;
10049}
10050
10051
10052#ifndef HMVMX_USE_FUNCTION_TABLE
10053DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10054{
10055#ifdef DEBUG_ramshankar
10056#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10057 do { \
10058 if (a_fSave != 0) \
10059 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10060 VBOXSTRICTRC rcStrict = a_CallExpr; \
10061 if (a_fSave != 0) \
10062 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10063 return rcStrict; \
10064 } while (0)
10065#else
10066# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10067#endif
10068 switch (rcReason)
10069 {
10070 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pVmxTransient));
10071 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pVmxTransient));
10072 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pVmxTransient));
10073 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pVmxTransient));
10074 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pVmxTransient));
10075 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pVmxTransient));
10076 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pVmxTransient));
10077 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pVmxTransient));
10078 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pVmxTransient));
10079 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pVmxTransient));
10080 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pVmxTransient));
10081 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pVmxTransient));
10082 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pVmxTransient));
10083 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pVmxTransient));
10084 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pVmxTransient));
10085 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pVmxTransient));
10086 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pVmxTransient));
10087 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pVmxTransient));
10088 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pVmxTransient));
10089 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pVmxTransient));
10090 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pVmxTransient));
10091 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pVmxTransient));
10092 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pVmxTransient));
10093 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pVmxTransient));
10094 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pVmxTransient));
10095 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pVmxTransient));
10096 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10097 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pVmxTransient));
10098 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pVmxTransient));
10099 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pVmxTransient));
10100 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pVmxTransient));
10101 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pVmxTransient));
10102 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pVmxTransient));
10103 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pVmxTransient));
10104
10105 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pVmxTransient);
10106 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pVmxTransient);
10107 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pVmxTransient);
10108 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pVmxTransient);
10109 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pVmxTransient);
10110 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pVmxTransient);
10111 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pVmxTransient);
10112 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pVmxTransient);
10113 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pVmxTransient);
10114
10115 case VMX_EXIT_VMCLEAR:
10116 case VMX_EXIT_VMLAUNCH:
10117 case VMX_EXIT_VMPTRLD:
10118 case VMX_EXIT_VMPTRST:
10119 case VMX_EXIT_VMREAD:
10120 case VMX_EXIT_VMRESUME:
10121 case VMX_EXIT_VMWRITE:
10122 case VMX_EXIT_VMXOFF:
10123 case VMX_EXIT_VMXON:
10124 case VMX_EXIT_INVEPT:
10125 case VMX_EXIT_INVVPID:
10126 case VMX_EXIT_VMFUNC:
10127 case VMX_EXIT_XSAVES:
10128 case VMX_EXIT_XRSTORS:
10129 return hmR0VmxExitSetPendingXcptUD(pVCpu, pVmxTransient);
10130
10131 case VMX_EXIT_ENCLS:
10132 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10133 case VMX_EXIT_PML_FULL:
10134 default:
10135 return hmR0VmxExitErrUndefined(pVCpu, pVmxTransient);
10136 }
10137#undef VMEXIT_CALL_RET
10138}
10139#endif /* !HMVMX_USE_FUNCTION_TABLE */
10140
10141
10142#ifdef VBOX_STRICT
10143/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10144# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10145 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10146
10147# define HMVMX_ASSERT_PREEMPT_CPUID() \
10148 do { \
10149 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10150 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10151 } while (0)
10152
10153# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10154 do { \
10155 AssertPtr((a_pVCpu)); \
10156 AssertPtr((a_pVmxTransient)); \
10157 Assert((a_pVmxTransient)->fVMEntryFailed == false); \
10158 Assert(ASMIntAreEnabled()); \
10159 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10160 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10161 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", (a_pVCpu)->idCpu)); \
10162 HMVMX_ASSERT_PREEMPT_SAFE(a_pVCpu); \
10163 if (VMMR0IsLogFlushDisabled((a_pVCpu))) \
10164 HMVMX_ASSERT_PREEMPT_CPUID(); \
10165 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10166 } while (0)
10167
10168# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10169 do { \
10170 Log4Func(("\n")); \
10171 } while (0)
10172#else
10173# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) \
10174 do { \
10175 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10176 NOREF((a_pVCpu)); NOREF((a_pVmxTransient)); \
10177 } while (0)
10178# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(a_pVCpu, a_pVmxTransient) do { } while (0)
10179#endif
10180
10181
10182/**
10183 * Advances the guest RIP by the specified number of bytes.
10184 *
10185 * @param pVCpu The cross context virtual CPU structure.
10186 * @param cbInstr Number of bytes to advance the RIP by.
10187 *
10188 * @remarks No-long-jump zone!!!
10189 */
10190DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, uint32_t cbInstr)
10191{
10192 /* Advance the RIP. */
10193 pVCpu->cpum.GstCtx.rip += cbInstr;
10194 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10195
10196 /* Update interrupt inhibition. */
10197 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10198 && pVCpu->cpum.GstCtx.rip != EMGetInhibitInterruptsPC(pVCpu))
10199 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10200}
10201
10202
10203/**
10204 * Advances the guest RIP after reading it from the VMCS.
10205 *
10206 * @returns VBox status code, no informational status codes.
10207 * @param pVCpu The cross context virtual CPU structure.
10208 * @param pVmxTransient Pointer to the VMX transient structure.
10209 *
10210 * @remarks No-long-jump zone!!!
10211 */
10212static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10213{
10214 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10215 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS);
10216 AssertRCReturn(rc, rc);
10217
10218 hmR0VmxAdvanceGuestRipBy(pVCpu, pVmxTransient->cbInstr);
10219
10220 /*
10221 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10222 * pending debug exception field as it takes care of priority of events.
10223 *
10224 * See Intel spec. 32.2.1 "Debug Exceptions".
10225 */
10226 if ( !pVCpu->hm.s.fSingleInstruction
10227 && pVCpu->cpum.GstCtx.eflags.Bits.u1TF)
10228 {
10229 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
10230 AssertRCReturn(rc, rc);
10231 }
10232
10233 return VINF_SUCCESS;
10234}
10235
10236
10237/**
10238 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10239 * and update error record fields accordingly.
10240 *
10241 * @return VMX_IGS_* return codes.
10242 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10243 * wrong with the guest state.
10244 *
10245 * @param pVCpu The cross context virtual CPU structure.
10246 *
10247 * @remarks This function assumes our cache of the VMCS controls
10248 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10249 */
10250static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu)
10251{
10252#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10253#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10254 uError = (err); \
10255 break; \
10256 } else do { } while (0)
10257
10258 int rc;
10259 PVM pVM = pVCpu->CTX_SUFF(pVM);
10260 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
10261 uint32_t uError = VMX_IGS_ERROR;
10262 uint32_t u32Val;
10263 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10264
10265 do
10266 {
10267 /*
10268 * CR0.
10269 */
10270 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10271 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10272 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10273 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10274 if (fUnrestrictedGuest)
10275 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10276
10277 uint32_t u32GuestCr0;
10278 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10279 AssertRCBreak(rc);
10280 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10281 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10282 if ( !fUnrestrictedGuest
10283 && (u32GuestCr0 & X86_CR0_PG)
10284 && !(u32GuestCr0 & X86_CR0_PE))
10285 {
10286 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10287 }
10288
10289 /*
10290 * CR4.
10291 */
10292 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10293 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10294
10295 uint32_t u32GuestCr4;
10296 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10297 AssertRCBreak(rc);
10298 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10299 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10300
10301 /*
10302 * IA32_DEBUGCTL MSR.
10303 */
10304 uint64_t u64Val;
10305 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10306 AssertRCBreak(rc);
10307 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10308 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10309 {
10310 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10311 }
10312 uint64_t u64DebugCtlMsr = u64Val;
10313
10314#ifdef VBOX_STRICT
10315 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10316 AssertRCBreak(rc);
10317 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10318#endif
10319 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_IA32E_MODE_GUEST);
10320
10321 /*
10322 * RIP and RFLAGS.
10323 */
10324 uint32_t u32Eflags;
10325#if HC_ARCH_BITS == 64
10326 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10327 AssertRCBreak(rc);
10328 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10329 if ( !fLongModeGuest
10330 || !pCtx->cs.Attr.n.u1Long)
10331 {
10332 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10333 }
10334 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10335 * must be identical if the "IA-32e mode guest" VM-entry
10336 * control is 1 and CS.L is 1. No check applies if the
10337 * CPU supports 64 linear-address bits. */
10338
10339 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10340 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10341 AssertRCBreak(rc);
10342 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10343 VMX_IGS_RFLAGS_RESERVED);
10344 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10345 u32Eflags = u64Val;
10346#else
10347 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10348 AssertRCBreak(rc);
10349 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10350 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10351#endif
10352
10353 if ( fLongModeGuest
10354 || ( fUnrestrictedGuest
10355 && !(u32GuestCr0 & X86_CR0_PE)))
10356 {
10357 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10358 }
10359
10360 uint32_t u32EntryInfo;
10361 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10362 AssertRCBreak(rc);
10363 if ( VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10364 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10365 {
10366 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10367 }
10368
10369 /*
10370 * 64-bit checks.
10371 */
10372#if HC_ARCH_BITS == 64
10373 if (fLongModeGuest)
10374 {
10375 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10376 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10377 }
10378
10379 if ( !fLongModeGuest
10380 && (u32GuestCr4 & X86_CR4_PCIDE))
10381 {
10382 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10383 }
10384
10385 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10386 * 51:32 beyond the processor's physical-address width are 0. */
10387
10388 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_DEBUG)
10389 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10390 {
10391 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10392 }
10393
10394 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10395 AssertRCBreak(rc);
10396 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10397
10398 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10399 AssertRCBreak(rc);
10400 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10401#endif
10402
10403 /*
10404 * PERF_GLOBAL MSR.
10405 */
10406 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PERF_MSR)
10407 {
10408 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10409 AssertRCBreak(rc);
10410 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10411 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10412 }
10413
10414 /*
10415 * PAT MSR.
10416 */
10417 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_PAT_MSR)
10418 {
10419 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10420 AssertRCBreak(rc);
10421 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10422 for (unsigned i = 0; i < 8; i++)
10423 {
10424 uint8_t u8Val = (u64Val & 0xff);
10425 if ( u8Val != 0 /* UC */
10426 && u8Val != 1 /* WC */
10427 && u8Val != 4 /* WT */
10428 && u8Val != 5 /* WP */
10429 && u8Val != 6 /* WB */
10430 && u8Val != 7 /* UC- */)
10431 {
10432 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10433 }
10434 u64Val >>= 8;
10435 }
10436 }
10437
10438 /*
10439 * EFER MSR.
10440 */
10441 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_LOAD_EFER_MSR)
10442 {
10443 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10444 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10445 AssertRCBreak(rc);
10446 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10447 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10448 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10449 & VMX_ENTRY_CTLS_IA32E_MODE_GUEST),
10450 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10451 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10452 || !(u32GuestCr0 & X86_CR0_PG)
10453 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10454 VMX_IGS_EFER_LMA_LME_MISMATCH);
10455 }
10456
10457 /*
10458 * Segment registers.
10459 */
10460 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10461 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10462 if (!(u32Eflags & X86_EFL_VM))
10463 {
10464 /* CS */
10465 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10466 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10467 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10468 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10469 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10470 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10471 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10472 /* CS cannot be loaded with NULL in protected mode. */
10473 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10474 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10475 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10476 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10477 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10478 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10479 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10480 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10481 else
10482 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10483
10484 /* SS */
10485 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10486 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10487 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10488 if ( !(pCtx->cr0 & X86_CR0_PE)
10489 || pCtx->cs.Attr.n.u4Type == 3)
10490 {
10491 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10492 }
10493 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10494 {
10495 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10496 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10497 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10498 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10499 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10500 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10501 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10502 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10503 }
10504
10505 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10506 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10507 {
10508 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10509 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10510 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10511 || pCtx->ds.Attr.n.u4Type > 11
10512 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10513 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10514 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10515 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10516 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10517 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10518 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10519 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10520 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10521 }
10522 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10523 {
10524 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10525 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10526 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10527 || pCtx->es.Attr.n.u4Type > 11
10528 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10529 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10530 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10531 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10532 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10533 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10534 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10535 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10536 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10537 }
10538 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10539 {
10540 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10541 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10542 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10543 || pCtx->fs.Attr.n.u4Type > 11
10544 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10545 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10546 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10547 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10548 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10549 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10550 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10551 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10552 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10553 }
10554 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10555 {
10556 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10557 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10558 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10559 || pCtx->gs.Attr.n.u4Type > 11
10560 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10561 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10562 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10563 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10564 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10565 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10566 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10567 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10568 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10569 }
10570 /* 64-bit capable CPUs. */
10571#if HC_ARCH_BITS == 64
10572 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10573 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10574 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10575 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10576 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10577 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10578 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10579 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10580 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10581 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10582 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10583#endif
10584 }
10585 else
10586 {
10587 /* V86 mode checks. */
10588 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10589 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10590 {
10591 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10592 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10593 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10594 }
10595 else
10596 {
10597 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10598 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10599 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10600 }
10601
10602 /* CS */
10603 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10604 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10605 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10606 /* SS */
10607 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10608 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10609 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10610 /* DS */
10611 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10612 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10613 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10614 /* ES */
10615 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10616 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10617 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10618 /* FS */
10619 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10620 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10621 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10622 /* GS */
10623 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10624 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10625 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10626 /* 64-bit capable CPUs. */
10627#if HC_ARCH_BITS == 64
10628 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10629 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10630 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10631 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10632 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10633 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10634 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10635 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10636 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10637 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10638 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10639#endif
10640 }
10641
10642 /*
10643 * TR.
10644 */
10645 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10646 /* 64-bit capable CPUs. */
10647#if HC_ARCH_BITS == 64
10648 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10649#endif
10650 if (fLongModeGuest)
10651 {
10652 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10653 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10654 }
10655 else
10656 {
10657 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10658 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10659 VMX_IGS_TR_ATTR_TYPE_INVALID);
10660 }
10661 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10662 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10663 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10664 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10665 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10666 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10667 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10668 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10669
10670 /*
10671 * GDTR and IDTR.
10672 */
10673#if HC_ARCH_BITS == 64
10674 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10675 AssertRCBreak(rc);
10676 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10677
10678 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10679 AssertRCBreak(rc);
10680 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10681#endif
10682
10683 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10684 AssertRCBreak(rc);
10685 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10686
10687 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10688 AssertRCBreak(rc);
10689 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10690
10691 /*
10692 * Guest Non-Register State.
10693 */
10694 /* Activity State. */
10695 uint32_t u32ActivityState;
10696 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10697 AssertRCBreak(rc);
10698 HMVMX_CHECK_BREAK( !u32ActivityState
10699 || (u32ActivityState & RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Misc, VMX_BF_MISC_ACTIVITY_STATES)),
10700 VMX_IGS_ACTIVITY_STATE_INVALID);
10701 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10702 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10703 uint32_t u32IntrState;
10704 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &u32IntrState);
10705 AssertRCBreak(rc);
10706 if ( u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS
10707 || u32IntrState == VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10708 {
10709 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10710 }
10711
10712 /** @todo Activity state and injecting interrupts. Left as a todo since we
10713 * currently don't use activity states but ACTIVE. */
10714
10715 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10716 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10717
10718 /* Guest interruptibility-state. */
10719 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10720 HMVMX_CHECK_BREAK((u32IntrState & (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS))
10721 != (VMX_VMCS_GUEST_INT_STATE_BLOCK_STI | VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10722 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10723 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10724 || !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10725 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10726 if (VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo))
10727 {
10728 if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_EXT_INT)
10729 {
10730 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10731 && !(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10732 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10733 }
10734 else if (VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10735 {
10736 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS),
10737 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10738 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI),
10739 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10740 }
10741 }
10742 /** @todo Assumes the processor is not in SMM. */
10743 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10744 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10745 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_ENTRY_CTLS_ENTRY_TO_SMM)
10746 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_SMI),
10747 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10748 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_PIN_CTLS_VIRT_NMI)
10749 && VMX_ENTRY_INT_INFO_IS_VALID(u32EntryInfo)
10750 && VMX_ENTRY_INT_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INT_INFO_TYPE_NMI)
10751 {
10752 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_NMI),
10753 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10754 }
10755
10756 /* Pending debug exceptions. */
10757#if HC_ARCH_BITS == 64
10758 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u64Val);
10759 AssertRCBreak(rc);
10760 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10761 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10762 u32Val = u64Val; /* For pending debug exceptions checks below. */
10763#else
10764 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_XCPTS, &u32Val);
10765 AssertRCBreak(rc);
10766 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10767 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10768#endif
10769
10770 if ( (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI)
10771 || (u32IntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_MOVSS)
10772 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10773 {
10774 if ( (u32Eflags & X86_EFL_TF)
10775 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10776 {
10777 /* Bit 14 is PendingDebug.BS. */
10778 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10779 }
10780 if ( !(u32Eflags & X86_EFL_TF)
10781 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10782 {
10783 /* Bit 14 is PendingDebug.BS. */
10784 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10785 }
10786 }
10787
10788 /* VMCS link pointer. */
10789 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10790 AssertRCBreak(rc);
10791 if (u64Val != UINT64_C(0xffffffffffffffff))
10792 {
10793 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10794 /** @todo Bits beyond the processor's physical-address width MBZ. */
10795 /** @todo 32-bit located in memory referenced by value of this field (as a
10796 * physical address) must contain the processor's VMCS revision ID. */
10797 /** @todo SMM checks. */
10798 }
10799
10800 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10801 * not using Nested Paging? */
10802 if ( pVM->hm.s.fNestedPaging
10803 && !fLongModeGuest
10804 && CPUMIsGuestInPAEModeEx(pCtx))
10805 {
10806 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10807 AssertRCBreak(rc);
10808 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10809
10810 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10811 AssertRCBreak(rc);
10812 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10813
10814 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10815 AssertRCBreak(rc);
10816 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10817
10818 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10821 }
10822
10823 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10824 if (uError == VMX_IGS_ERROR)
10825 uError = VMX_IGS_REASON_NOT_FOUND;
10826 } while (0);
10827
10828 pVCpu->hm.s.u32HMError = uError;
10829 return uError;
10830
10831#undef HMVMX_ERROR_BREAK
10832#undef HMVMX_CHECK_BREAK
10833}
10834
10835/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10836/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10837/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10838
10839/** @name VM-exit handlers.
10840 * @{
10841 */
10842
10843/**
10844 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10845 */
10846HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10847{
10848 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10849 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10850 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10851 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10852 return VINF_SUCCESS;
10853 return VINF_EM_RAW_INTERRUPT;
10854}
10855
10856
10857/**
10858 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10859 */
10860HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10861{
10862 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10863 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10864
10865 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10866 AssertRCReturn(rc, rc);
10867
10868 uint32_t uIntType = VMX_EXIT_INT_INFO_TYPE(pVmxTransient->uExitIntInfo);
10869 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_EXIT_CTLS_ACK_EXT_INT)
10870 && uIntType != VMX_EXIT_INT_INFO_TYPE_EXT_INT);
10871 Assert(VMX_EXIT_INT_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10872
10873 if (uIntType == VMX_EXIT_INT_INFO_TYPE_NMI)
10874 {
10875 /*
10876 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10877 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10878 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10879 *
10880 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10881 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10882 */
10883 VMXDispatchHostNmi();
10884 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10885 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10886 return VINF_SUCCESS;
10887 }
10888
10889 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10890 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
10891 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10892 { /* likely */ }
10893 else
10894 {
10895 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10896 rcStrictRc1 = VINF_SUCCESS;
10897 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10898 return rcStrictRc1;
10899 }
10900
10901 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10902 uint32_t uVector = VMX_EXIT_INT_INFO_VECTOR(uExitIntInfo);
10903 switch (uIntType)
10904 {
10905 case VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10906 Assert(uVector == X86_XCPT_DB);
10907 RT_FALL_THRU();
10908 case VMX_EXIT_INT_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10909 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INT_INFO_TYPE_PRIV_SW_XCPT);
10910 RT_FALL_THRU();
10911 case VMX_EXIT_INT_INFO_TYPE_HW_XCPT:
10912 {
10913 /*
10914 * If there's any exception caused as a result of event injection, the resulting
10915 * secondary/final execption will be pending, we shall continue guest execution
10916 * after injecting the event. The page-fault case is complicated and we manually
10917 * handle any currently pending event in hmR0VmxExitXcptPF.
10918 */
10919 if (!pVCpu->hm.s.Event.fPending)
10920 { /* likely */ }
10921 else if (uVector != X86_XCPT_PF)
10922 {
10923 rc = VINF_SUCCESS;
10924 break;
10925 }
10926
10927 switch (uVector)
10928 {
10929 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pVmxTransient); break;
10930 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pVmxTransient); break;
10931 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pVmxTransient); break;
10932 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pVmxTransient); break;
10933 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pVmxTransient); break;
10934 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pVmxTransient); break;
10935
10936 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
10937 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10938 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
10939 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10940 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
10941 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10942 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
10943 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10944 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
10945 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10946 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
10947 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10948 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
10949 rc = hmR0VmxExitXcptGeneric(pVCpu, pVmxTransient); break;
10950 default:
10951 {
10952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
10953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10954 {
10955 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
10956 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
10957 Assert(CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx));
10958
10959 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
10960 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10961 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
10962 AssertRCReturn(rc, rc);
10963 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
10964 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
10965 0 /* GCPtrFaultAddress */);
10966 }
10967 else
10968 {
10969 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
10970 pVCpu->hm.s.u32HMError = uVector;
10971 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
10972 }
10973 break;
10974 }
10975 }
10976 break;
10977 }
10978
10979 default:
10980 {
10981 pVCpu->hm.s.u32HMError = uExitIntInfo;
10982 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
10983 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INT_INFO_TYPE(uExitIntInfo)));
10984 break;
10985 }
10986 }
10987 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10988 return rc;
10989}
10990
10991
10992/**
10993 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
10994 */
10995HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
10996{
10997 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
10998
10999 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11000 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11001
11002 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11003 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11004 return VINF_SUCCESS;
11005}
11006
11007
11008/**
11009 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11010 */
11011HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11012{
11013 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11014 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_NMI_WINDOW_EXIT)))
11015 {
11016 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11017 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11018 }
11019
11020 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11021
11022 /*
11023 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11024 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11025 */
11026 uint32_t fIntrState = 0;
11027 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11028 AssertRCReturn(rc, rc);
11029
11030 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INT_STATE_BLOCK_STI);
11031 if ( fBlockSti
11032 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11033 {
11034 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11035 }
11036
11037 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11038 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11039
11040 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11041 return VINF_SUCCESS;
11042}
11043
11044
11045/**
11046 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11047 */
11048HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11049{
11050 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11051 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11052}
11053
11054
11055/**
11056 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11057 */
11058HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11059{
11060 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11061 return hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11062}
11063
11064
11065/**
11066 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11067 */
11068HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11069{
11070 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11071
11072 /*
11073 * Get the state we need and update the exit history entry.
11074 */
11075 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11076 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_RAX | CPUMCTX_EXTRN_RCX);
11077 AssertRCReturn(rc, rc);
11078
11079 VBOXSTRICTRC rcStrict;
11080 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11081 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11082 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11083 if (!pExitRec)
11084 {
11085 /*
11086 * Regular CPUID instruction execution.
11087 */
11088 rcStrict = IEMExecDecodedCpuid(pVCpu, pVmxTransient->cbInstr);
11089 if (rcStrict == VINF_SUCCESS)
11090 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RAX
11091 | HM_CHANGED_GUEST_RCX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RBX);
11092 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11093 {
11094 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11095 rcStrict = VINF_SUCCESS;
11096 }
11097 }
11098 else
11099 {
11100 /*
11101 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11102 */
11103 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11104 AssertRCReturn(rc2, rc2);
11105
11106 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11107 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11108
11109 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11110 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11111
11112 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11113 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11114 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11115 }
11116 return rcStrict;
11117}
11118
11119
11120/**
11121 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11122 */
11123HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11124{
11125 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11126 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11127 AssertRCReturn(rc, rc);
11128
11129 if (pVCpu->cpum.GstCtx.cr4 & X86_CR4_SMXE)
11130 return VINF_EM_RAW_EMULATE_INSTR;
11131
11132 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11133 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11134}
11135
11136
11137/**
11138 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11139 */
11140HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11141{
11142 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11143 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11144 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11145 AssertRCReturn(rc, rc);
11146
11147 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11148 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11149 {
11150 /* If we get a spurious VM-exit when offsetting is enabled,
11151 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11152 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11153 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11154 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11155 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11156 }
11157 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11158 {
11159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11160 rcStrict = VINF_SUCCESS;
11161 }
11162 return rcStrict;
11163}
11164
11165
11166/**
11167 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11168 */
11169HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11170{
11171 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11172 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_TSC_AUX);
11173 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11174 AssertRCReturn(rc, rc);
11175
11176 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11177 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11178 {
11179 /* If we get a spurious VM-exit when offsetting is enabled,
11180 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11181 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TSC_OFFSETTING)
11182 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11183 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11184 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX | HM_CHANGED_GUEST_RCX);
11185 }
11186 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11187 {
11188 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11189 rcStrict = VINF_SUCCESS;
11190 }
11191 return rcStrict;
11192}
11193
11194
11195/**
11196 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11197 */
11198HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11199{
11200 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11201 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4 | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11202 AssertRCReturn(rc, rc);
11203
11204 PVM pVM = pVCpu->CTX_SUFF(pVM);
11205 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11206 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11207 if (RT_LIKELY(rc == VINF_SUCCESS))
11208 {
11209 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11210 Assert(pVmxTransient->cbInstr == 2);
11211 }
11212 else
11213 {
11214 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11215 rc = VERR_EM_INTERPRETER;
11216 }
11217 return rc;
11218}
11219
11220
11221/**
11222 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11223 */
11224HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11225{
11226 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11227
11228 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11229 if (EMAreHypercallInstructionsEnabled(pVCpu))
11230 {
11231 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_SS
11232 | CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_EFER);
11233 AssertRCReturn(rc, rc);
11234
11235 /* Perform the hypercall. */
11236 rcStrict = GIMHypercall(pVCpu, &pVCpu->cpum.GstCtx);
11237 if (rcStrict == VINF_SUCCESS)
11238 {
11239 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11240 AssertRCReturn(rc, rc);
11241 }
11242 else
11243 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11244 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11245 || RT_FAILURE(rcStrict));
11246
11247 /* If the hypercall changes anything other than guest's general-purpose registers,
11248 we would need to reload the guest changed bits here before VM-entry. */
11249 }
11250 else
11251 Log4Func(("Hypercalls not enabled\n"));
11252
11253 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11254 if (RT_FAILURE(rcStrict))
11255 {
11256 hmR0VmxSetPendingXcptUD(pVCpu);
11257 rcStrict = VINF_SUCCESS;
11258 }
11259
11260 return rcStrict;
11261}
11262
11263
11264/**
11265 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11266 */
11267HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11268{
11269 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11270 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11271
11272 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11273 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11274 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_MEM_MASK | CPUMCTX_EXTRN_DS);
11275 AssertRCReturn(rc, rc);
11276
11277 VBOXSTRICTRC rcStrict = IEMExecDecodedInvlpg(pVCpu, pVmxTransient->cbInstr, pVmxTransient->uExitQualification);
11278
11279 if (rcStrict == VINF_SUCCESS || rcStrict == VINF_PGM_SYNC_CR3)
11280 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11281 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11282 {
11283 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11284 rcStrict = VINF_SUCCESS;
11285 }
11286 else
11287 AssertMsgFailed(("Unexpected IEMExecDecodedInvlpg(%#RX64) sttus: %Rrc\n",
11288 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11289 return rcStrict;
11290}
11291
11292
11293/**
11294 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11295 */
11296HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11297{
11298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11299 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11300 AssertRCReturn(rc, rc);
11301
11302 PVM pVM = pVCpu->CTX_SUFF(pVM);
11303 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11304 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11305 if (RT_LIKELY(rc == VINF_SUCCESS))
11306 rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11307 else
11308 {
11309 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11310 rc = VERR_EM_INTERPRETER;
11311 }
11312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11313 return rc;
11314}
11315
11316
11317/**
11318 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11319 */
11320HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11321{
11322 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11323 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_RFLAGS | CPUMCTX_EXTRN_SS);
11324 AssertRCReturn(rc, rc);
11325
11326 PVM pVM = pVCpu->CTX_SUFF(pVM);
11327 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11328 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pCtx));
11329 rc = VBOXSTRICTRC_VAL(rc2);
11330 if (RT_LIKELY( rc == VINF_SUCCESS
11331 || rc == VINF_EM_HALT))
11332 {
11333 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11334 AssertRCReturn(rc3, rc3);
11335
11336 if ( rc == VINF_EM_HALT
11337 && EMMonitorWaitShouldContinue(pVCpu, pCtx))
11338 rc = VINF_SUCCESS;
11339 }
11340 else
11341 {
11342 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11343 rc = VERR_EM_INTERPRETER;
11344 }
11345 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11346 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11347 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11348 return rc;
11349}
11350
11351
11352/**
11353 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11354 */
11355HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11356{
11357 /*
11358 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11359 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11360 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11361 * VMX root operation. If we get here, something funny is going on.
11362 *
11363 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11364 */
11365 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11366 AssertMsgFailed(("Unexpected RSM VM-exit\n"));
11367 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11368}
11369
11370
11371/**
11372 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11373 */
11374HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11375{
11376 /*
11377 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11378 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11379 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11380 * an SMI. If we get here, something funny is going on.
11381 *
11382 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11383 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11384 */
11385 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11386 AssertMsgFailed(("Unexpected SMI VM-exit\n"));
11387 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11388}
11389
11390
11391/**
11392 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11393 */
11394HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11395{
11396 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11397 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11398 AssertMsgFailed(("Unexpected IO SMI VM-exit\n"));
11399 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11400}
11401
11402
11403/**
11404 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11405 */
11406HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11407{
11408 /*
11409 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11410 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11411 * See Intel spec. 25.3 "Other Causes of VM-exits".
11412 */
11413 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11414 AssertMsgFailed(("Unexpected SIPI VM-exit\n"));
11415 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11416}
11417
11418
11419/**
11420 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11421 * VM-exit.
11422 */
11423HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11424{
11425 /*
11426 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11427 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11428 *
11429 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11430 * See Intel spec. "23.8 Restrictions on VMX operation".
11431 */
11432 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11433 return VINF_SUCCESS;
11434}
11435
11436
11437/**
11438 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11439 * VM-exit.
11440 */
11441HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11442{
11443 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11444 return VINF_EM_RESET;
11445}
11446
11447
11448/**
11449 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11450 */
11451HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11452{
11453 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11454 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_HLT_EXIT);
11455
11456 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
11457 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
11458 AssertRCReturn(rc, rc);
11459
11460 if (EMShouldContinueAfterHalt(pVCpu, &pVCpu->cpum.GstCtx)) /* Requires eflags. */
11461 rc = VINF_SUCCESS;
11462 else
11463 rc = VINF_EM_HALT;
11464
11465 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11466 if (rc != VINF_SUCCESS)
11467 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11468 return rc;
11469}
11470
11471
11472/**
11473 * VM-exit handler for instructions that result in a \#UD exception delivered to
11474 * the guest.
11475 */
11476HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11477{
11478 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11479 hmR0VmxSetPendingXcptUD(pVCpu);
11480 return VINF_SUCCESS;
11481}
11482
11483
11484/**
11485 * VM-exit handler for expiry of the VMX preemption timer.
11486 */
11487HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11488{
11489 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11490
11491 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11492 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11493
11494 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11495 PVM pVM = pVCpu->CTX_SUFF(pVM);
11496 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11497 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11498 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11499}
11500
11501
11502/**
11503 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11504 */
11505HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11506{
11507 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11508
11509 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11510 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_CR4);
11511 AssertRCReturn(rc, rc);
11512
11513 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11514 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11515 : HM_CHANGED_RAISED_XCPT_MASK);
11516
11517 PCCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
11518 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pCtx->cr4 & X86_CR4_OSXSAVE) && pCtx->aXcr[0] != ASMGetXcr0();
11519
11520 return rcStrict;
11521}
11522
11523
11524/**
11525 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11526 */
11527HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11528{
11529 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11530 /** @todo Use VM-exit instruction information. */
11531 return VERR_EM_INTERPRETER;
11532}
11533
11534
11535/**
11536 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11537 * Error VM-exit.
11538 */
11539HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11540{
11541 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11542 AssertRCReturn(rc, rc);
11543 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11544 if (RT_FAILURE(rc))
11545 return rc;
11546
11547 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu);
11548 NOREF(uInvalidReason);
11549
11550#ifdef VBOX_STRICT
11551 uint32_t fIntrState;
11552 RTHCUINTREG uHCReg;
11553 uint64_t u64Val;
11554 uint32_t u32Val;
11555
11556 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11557 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11558 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11559 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INT_STATE, &fIntrState);
11560 AssertRCReturn(rc, rc);
11561
11562 Log4(("uInvalidReason %u\n", uInvalidReason));
11563 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11564 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11565 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11566 Log4(("VMX_VMCS32_GUEST_INT_STATE %#RX32\n", fIntrState));
11567
11568 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11569 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11570 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11571 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11572 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11573 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11574 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11575 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11576 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11577 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11578 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11579 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11580
11581 hmR0DumpRegs(pVCpu);
11582#else
11583 NOREF(pVmxTransient);
11584#endif
11585
11586 return VERR_VMX_INVALID_GUEST_STATE;
11587}
11588
11589
11590/**
11591 * VM-exit handler for VM-entry failure due to an MSR-load
11592 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11593 */
11594HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11595{
11596 AssertMsgFailed(("Unexpected MSR-load exit\n"));
11597 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11598}
11599
11600
11601/**
11602 * VM-exit handler for VM-entry failure due to a machine-check event
11603 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11604 */
11605HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11606{
11607 AssertMsgFailed(("Unexpected machine-check event exit\n"));
11608 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11609}
11610
11611
11612/**
11613 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11614 * theory.
11615 */
11616HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11617{
11618 RT_NOREF2(pVCpu, pVmxTransient);
11619 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d\n", pVmxTransient->uExitReason));
11620 return VERR_VMX_UNDEFINED_EXIT_CODE;
11621}
11622
11623
11624/**
11625 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11626 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11627 * Conditional VM-exit.
11628 */
11629HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11630{
11631 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11632
11633 /* By default, we don't enable VMX_PROC_CTLS2_DESCRIPTOR_TABLE_EXIT. */
11634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11635 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_DESC_TABLE_EXIT)
11636 return VERR_EM_INTERPRETER;
11637 AssertMsgFailed(("Unexpected XDTR access\n"));
11638 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11639}
11640
11641
11642/**
11643 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11644 */
11645HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11646{
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11648
11649 /* By default, we don't enable VMX_PROC_CTLS2_RDRAND_EXIT. */
11650 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_PROC_CTLS2_RDRAND_EXIT)
11651 return VERR_EM_INTERPRETER;
11652 AssertMsgFailed(("Unexpected RDRAND exit\n"));
11653 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11654}
11655
11656
11657/**
11658 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11659 */
11660HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11661{
11662 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11663
11664 /** @todo Optimize this: We currently drag in in the whole MSR state
11665 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11666 * MSRs required. That would require changes to IEM and possibly CPUM too.
11667 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11668 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; NOREF(idMsr); /* Save it. */
11669 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11670 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11671 AssertRCReturn(rc, rc);
11672
11673 Log4Func(("ecx=%#RX32\n", idMsr));
11674
11675#ifdef VBOX_STRICT
11676 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS)
11677 {
11678 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr)
11679 && idMsr != MSR_K6_EFER)
11680 {
11681 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n", idMsr));
11682 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11683 }
11684 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11685 {
11686 VMXMSREXITREAD enmRead;
11687 VMXMSREXITWRITE enmWrite;
11688 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11689 AssertRCReturn(rc2, rc2);
11690 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11691 {
11692 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", idMsr));
11693 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11694 }
11695 }
11696 }
11697#endif
11698
11699 VBOXSTRICTRC rcStrict = IEMExecDecodedRdmsr(pVCpu, pVmxTransient->cbInstr);
11700 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11701 if (rcStrict == VINF_SUCCESS)
11702 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11703 | HM_CHANGED_GUEST_RAX | HM_CHANGED_GUEST_RDX);
11704 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11705 {
11706 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11707 rcStrict = VINF_SUCCESS;
11708 }
11709 else
11710 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_READ, ("Unexpected IEMExecDecodedRdmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11711
11712 return rcStrict;
11713}
11714
11715
11716/**
11717 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11718 */
11719HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11720{
11721 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11722
11723 /** @todo Optimize this: We currently drag in in the whole MSR state
11724 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11725 * MSRs required. That would require changes to IEM and possibly CPUM too.
11726 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11727 uint32_t const idMsr = pVCpu->cpum.GstCtx.ecx; /* Save it. */
11728 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11729 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11730 AssertRCReturn(rc, rc);
11731
11732 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pVCpu->cpum.GstCtx.edx, pVCpu->cpum.GstCtx.eax));
11733
11734 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11735 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11736
11737 if (rcStrict == VINF_SUCCESS)
11738 {
11739 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
11740
11741 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11742 if ( idMsr == MSR_IA32_APICBASE
11743 || ( idMsr >= MSR_IA32_X2APIC_START
11744 && idMsr <= MSR_IA32_X2APIC_END))
11745 {
11746 /*
11747 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11748 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11749 */
11750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11751 }
11752 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11753 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11754 else if (idMsr == MSR_K6_EFER)
11755 {
11756 /*
11757 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11758 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11759 * the other bits as well, SCE and NXE. See @bugref{7368}.
11760 */
11761 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR | HM_CHANGED_VMX_ENTRY_CTLS
11762 | HM_CHANGED_VMX_EXIT_CTLS);
11763 }
11764
11765 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11766 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_MSR_BITMAPS))
11767 {
11768 switch (idMsr)
11769 {
11770 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11771 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11772 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11773 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11774 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11775 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11776 default:
11777 {
11778 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11779 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11780 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11781 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11782 break;
11783 }
11784 }
11785 }
11786#ifdef VBOX_STRICT
11787 else
11788 {
11789 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11790 switch (idMsr)
11791 {
11792 case MSR_IA32_SYSENTER_CS:
11793 case MSR_IA32_SYSENTER_EIP:
11794 case MSR_IA32_SYSENTER_ESP:
11795 case MSR_K8_FS_BASE:
11796 case MSR_K8_GS_BASE:
11797 {
11798 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11799 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11800 }
11801
11802 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11803 default:
11804 {
11805 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11806 {
11807 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11808 if (idMsr != MSR_K6_EFER)
11809 {
11810 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11811 idMsr));
11812 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11813 }
11814 }
11815
11816 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11817 {
11818 VMXMSREXITREAD enmRead;
11819 VMXMSREXITWRITE enmWrite;
11820 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11821 AssertRCReturn(rc2, rc2);
11822 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11823 {
11824 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11825 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11826 }
11827 }
11828 break;
11829 }
11830 }
11831 }
11832#endif /* VBOX_STRICT */
11833 }
11834 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11835 {
11836 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
11837 rcStrict = VINF_SUCCESS;
11838 }
11839 else
11840 AssertMsg(rcStrict == VINF_CPUM_R3_MSR_WRITE, ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11841
11842 return rcStrict;
11843}
11844
11845
11846/**
11847 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11848 */
11849HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11850{
11851 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11852 /** @todo The guest has likely hit a contended spinlock. We might want to
11853 * poke a schedule different guest VCPU. */
11854 return VINF_EM_RAW_INTERRUPT;
11855}
11856
11857
11858/**
11859 * VM-exit handler for when the TPR value is lowered below the specified
11860 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11861 */
11862HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11863{
11864 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11865 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW);
11866
11867 /*
11868 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11869 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11870 */
11871 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11872 return VINF_SUCCESS;
11873}
11874
11875
11876/**
11877 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11878 * VM-exit.
11879 *
11880 * @retval VINF_SUCCESS when guest execution can continue.
11881 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11882 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11883 * interpreter.
11884 */
11885HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
11886{
11887 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
11888 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11889
11890 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11891 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11892 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11893 AssertRCReturn(rc, rc);
11894
11895 VBOXSTRICTRC rcStrict;
11896 PVM pVM = pVCpu->CTX_SUFF(pVM);
11897 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11898 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11899 switch (uAccessType)
11900 {
11901 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11902 {
11903 uint32_t const uOldCr0 = pVCpu->cpum.GstCtx.cr0;
11904 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11905 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11906 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11907 AssertMsg( rcStrict == VINF_SUCCESS
11908 || rcStrict == VINF_IEM_RAISED_XCPT
11909 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11910
11911 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11912 {
11913 case 0:
11914 {
11915 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11916 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
11917 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11918 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr0));
11919
11920 /*
11921 * This is a kludge for handling switches back to real mode when we try to use
11922 * V86 mode to run real mode code directly. Problem is that V86 mode cannot
11923 * deal with special selector values, so we have to return to ring-3 and run
11924 * there till the selector values are V86 mode compatible.
11925 *
11926 * Note! Using VINF_EM_RESCHEDULE_REM here rather than VINF_EM_RESCHEDULE since the
11927 * latter is an alias for VINF_IEM_RAISED_XCPT which is converted to VINF_SUCCESs
11928 * at the end of this function.
11929 */
11930 if ( rc == VINF_SUCCESS
11931 && !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
11932 && CPUMIsGuestInRealModeEx(&pVCpu->cpum.GstCtx)
11933 && (uOldCr0 & X86_CR0_PE)
11934 && !(pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE) )
11935 {
11936 /** @todo check selectors rather than returning all the time. */
11937 Log4(("CRx CR0 write: back to real mode -> VINF_EM_RESCHEDULE_REM\n"));
11938 rcStrict = VINF_EM_RESCHEDULE_REM;
11939 }
11940 break;
11941 }
11942
11943 case 2:
11944 {
11945 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11946 /* Nothing to do here, CR2 it's not part of the VMCS. */
11947 break;
11948 }
11949
11950 case 3:
11951 {
11952 Assert( !pVM->hm.s.fNestedPaging
11953 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11954 || pVCpu->hm.s.fUsingDebugLoop);
11955 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
11956 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11957 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR3);
11958 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cr3));
11959 break;
11960 }
11961
11962 case 4:
11963 {
11964 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
11965 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11966 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR4);
11967 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
11968 pVCpu->cpum.GstCtx.cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
11969 break;
11970 }
11971
11972 case 8:
11973 {
11974 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
11975 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
11976 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged,
11977 HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_APIC_TPR);
11978 break;
11979 }
11980 default:
11981 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
11982 break;
11983 }
11984 break;
11985 }
11986
11987 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
11988 {
11989 Assert( !pVM->hm.s.fNestedPaging
11990 || !CPUMIsGuestPagingEnabledEx(&pVCpu->cpum.GstCtx)
11991 || pVCpu->hm.s.fUsingDebugLoop
11992 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
11993 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
11994 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
11995 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW));
11996
11997 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
11998 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
11999 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12000 AssertMsg( rcStrict == VINF_SUCCESS
12001 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12002#ifdef VBOX_WITH_STATISTICS
12003 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12004 {
12005 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12006 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12007 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12008 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12009 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12010 }
12011#endif
12012 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12013 VBOXSTRICTRC_VAL(rcStrict)));
12014 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12015 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_RSP);
12016 else
12017 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12018 break;
12019 }
12020
12021 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12022 {
12023 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12024 AssertMsg( rcStrict == VINF_SUCCESS
12025 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12026
12027 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12028 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12029 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12030 break;
12031 }
12032
12033 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12034 {
12035 /* Note! LMSW cannot clear CR0.PE, so no fRealOnV86Active kludge needed here. */
12036 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12037 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12038 AssertMsg( rcStrict == VINF_SUCCESS
12039 || rcStrict == VINF_IEM_RAISED_XCPT
12040 , ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12041
12042 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS | HM_CHANGED_GUEST_CR0);
12043 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12044 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12045 break;
12046 }
12047
12048 default:
12049 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12050 VERR_VMX_UNEXPECTED_EXCEPTION);
12051 }
12052
12053 Assert( (pVCpu->hm.s.fCtxChanged & (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS))
12054 == (HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS));
12055 if (rcStrict == VINF_IEM_RAISED_XCPT)
12056 {
12057 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_RAISED_XCPT_MASK);
12058 rcStrict = VINF_SUCCESS;
12059 }
12060
12061 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12062 NOREF(pVM);
12063 return rcStrict;
12064}
12065
12066
12067/**
12068 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12069 * VM-exit.
12070 */
12071HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12072{
12073 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12074 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12075
12076 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12077 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12078 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12079 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK | CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_EFER);
12080 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12081 AssertRCReturn(rc, rc);
12082
12083 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12084 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12085 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12086 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12087 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12088 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12089 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12090 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12091 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12092
12093 /*
12094 * Update exit history to see if this exit can be optimized.
12095 */
12096 VBOXSTRICTRC rcStrict;
12097 PCEMEXITREC pExitRec = NULL;
12098 if ( !fGstStepping
12099 && !fDbgStepping)
12100 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12101 !fIOString
12102 ? !fIOWrite
12103 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12104 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12105 : !fIOWrite
12106 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12107 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12108 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12109 if (!pExitRec)
12110 {
12111 /* I/O operation lookup arrays. */
12112 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12113 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12114 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12115 uint32_t const cbInstr = pVmxTransient->cbInstr;
12116 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12117 PVM pVM = pVCpu->CTX_SUFF(pVM);
12118 if (fIOString)
12119 {
12120 /*
12121 * INS/OUTS - I/O String instruction.
12122 *
12123 * Use instruction-information if available, otherwise fall back on
12124 * interpreting the instruction.
12125 */
12126 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12127 AssertReturn(pCtx->dx == uIOPort, VERR_VMX_IPE_2);
12128 bool const fInsOutsInfo = RT_BF_GET(pVM->hm.s.vmx.Msrs.u64Basic, VMX_BF_BASIC_VMCS_INS_OUTS);
12129 if (fInsOutsInfo)
12130 {
12131 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12132 AssertRCReturn(rc2, rc2);
12133 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12134 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12135 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12136 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12137 if (fIOWrite)
12138 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12139 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12140 else
12141 {
12142 /*
12143 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12144 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12145 * See Intel Instruction spec. for "INS".
12146 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12147 */
12148 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12149 }
12150 }
12151 else
12152 rcStrict = IEMExecOne(pVCpu);
12153
12154 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12155 fUpdateRipAlready = true;
12156 }
12157 else
12158 {
12159 /*
12160 * IN/OUT - I/O instruction.
12161 */
12162 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pCtx->cs.Sel, pCtx->rip, uIOPort, cbValue, fIOWrite ? 'w' : 'r'));
12163 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12164 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12165 if (fIOWrite)
12166 {
12167 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pCtx->eax & uAndVal, cbValue);
12168 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12169 if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12170 && !pCtx->eflags.Bits.u1TF)
12171 rcStrict = EMRZSetPendingIoPortWrite(pVCpu, uIOPort, cbInstr, cbValue, pCtx->eax & uAndVal);
12172 }
12173 else
12174 {
12175 uint32_t u32Result = 0;
12176 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12177 if (IOM_SUCCESS(rcStrict))
12178 {
12179 /* Save result of I/O IN instr. in AL/AX/EAX. */
12180 pCtx->eax = (pCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12181 }
12182 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12183 && !pCtx->eflags.Bits.u1TF)
12184 rcStrict = EMRZSetPendingIoPortRead(pVCpu, uIOPort, cbInstr, cbValue);
12185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12186 }
12187 }
12188
12189 if (IOM_SUCCESS(rcStrict))
12190 {
12191 if (!fUpdateRipAlready)
12192 {
12193 hmR0VmxAdvanceGuestRipBy(pVCpu, cbInstr);
12194 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12195 }
12196
12197 /*
12198 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12199 * while booting Fedora 17 64-bit guest.
12200 *
12201 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12202 */
12203 if (fIOString)
12204 {
12205 /** @todo Single-step for INS/OUTS with REP prefix? */
12206 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12207 }
12208 else if ( !fDbgStepping
12209 && fGstStepping)
12210 {
12211 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12212 AssertRCReturn(rc, rc);
12213 }
12214
12215 /*
12216 * If any I/O breakpoints are armed, we need to check if one triggered
12217 * and take appropriate action.
12218 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12219 */
12220 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12221 AssertRCReturn(rc, rc);
12222
12223 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12224 * execution engines about whether hyper BPs and such are pending. */
12225 uint32_t const uDr7 = pCtx->dr[7];
12226 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12227 && X86_DR7_ANY_RW_IO(uDr7)
12228 && (pCtx->cr4 & X86_CR4_DE))
12229 || DBGFBpIsHwIoArmed(pVM)))
12230 {
12231 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12232
12233 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12234 VMMRZCallRing3Disable(pVCpu);
12235 HM_DISABLE_PREEMPT(pVCpu);
12236
12237 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12238
12239 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pCtx, uIOPort, cbValue);
12240 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12241 {
12242 /* Raise #DB. */
12243 if (fIsGuestDbgActive)
12244 ASMSetDR6(pCtx->dr[6]);
12245 if (pCtx->dr[7] != uDr7)
12246 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12247
12248 hmR0VmxSetPendingXcptDB(pVCpu);
12249 }
12250 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12251 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12252 else if ( rcStrict2 != VINF_SUCCESS
12253 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12254 rcStrict = rcStrict2;
12255 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12256
12257 HM_RESTORE_PREEMPT();
12258 VMMRZCallRing3Enable(pVCpu);
12259 }
12260 }
12261
12262#ifdef VBOX_STRICT
12263 if ( rcStrict == VINF_IOM_R3_IOPORT_READ
12264 || rcStrict == VINF_EM_PENDING_R3_IOPORT_READ)
12265 Assert(!fIOWrite);
12266 else if ( rcStrict == VINF_IOM_R3_IOPORT_WRITE
12267 || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE
12268 || rcStrict == VINF_EM_PENDING_R3_IOPORT_WRITE)
12269 Assert(fIOWrite);
12270 else
12271 {
12272# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12273 * statuses, that the VMM device and some others may return. See
12274 * IOM_SUCCESS() for guidance. */
12275 AssertMsg( RT_FAILURE(rcStrict)
12276 || rcStrict == VINF_SUCCESS
12277 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12278 || rcStrict == VINF_EM_DBG_BREAKPOINT
12279 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12280 || rcStrict == VINF_EM_RAW_TO_R3
12281 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12282# endif
12283 }
12284#endif
12285 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12286 }
12287 else
12288 {
12289 /*
12290 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12291 */
12292 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12293 AssertRCReturn(rc2, rc2);
12294 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12295 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12296 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12297 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12298 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12299 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12300
12301 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12302 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12303
12304 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12305 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12306 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12307 }
12308 return rcStrict;
12309}
12310
12311
12312/**
12313 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12314 * VM-exit.
12315 */
12316HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12317{
12318 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12319
12320 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12321 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12322 AssertRCReturn(rc, rc);
12323 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12324 {
12325 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12326 AssertRCReturn(rc, rc);
12327 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12328 {
12329 uint32_t uErrCode;
12330 RTGCUINTPTR GCPtrFaultAddress;
12331 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12332 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12333 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_IS_ERROR_CODE_VALID(pVmxTransient->uIdtVectoringInfo);
12334 if (fErrorCodeValid)
12335 {
12336 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12337 AssertRCReturn(rc, rc);
12338 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12339 }
12340 else
12341 uErrCode = 0;
12342
12343 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12344 && uVector == X86_XCPT_PF)
12345 GCPtrFaultAddress = pVCpu->cpum.GstCtx.cr2;
12346 else
12347 GCPtrFaultAddress = 0;
12348
12349 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12350 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12351
12352 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12353 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12354 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12355 }
12356 }
12357
12358 /* Fall back to the interpreter to emulate the task-switch. */
12359 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12360 return VERR_EM_INTERPRETER;
12361}
12362
12363
12364/**
12365 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12366 */
12367HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12368{
12369 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12370 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_MONITOR_TRAP_FLAG);
12371 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MONITOR_TRAP_FLAG;
12372 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12373 AssertRCReturn(rc, rc);
12374 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12375 return VINF_EM_DBG_STEPPED;
12376}
12377
12378
12379/**
12380 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12381 */
12382HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12383{
12384 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12385
12386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12387
12388 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12389 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12390 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12391 {
12392 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12393 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12394 {
12395 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12396 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12397 }
12398 }
12399 else
12400 {
12401 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12402 rcStrict1 = VINF_SUCCESS;
12403 return rcStrict1;
12404 }
12405
12406 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12407 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12408 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12409 AssertRCReturn(rc, rc);
12410
12411 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12412 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12413 VBOXSTRICTRC rcStrict2;
12414 switch (uAccessType)
12415 {
12416 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12417 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12418 {
12419 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_PROC_CTLS_USE_TPR_SHADOW)
12420 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12421 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12422
12423 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12424 GCPhys &= PAGE_BASE_GC_MASK;
12425 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12426 PVM pVM = pVCpu->CTX_SUFF(pVM);
12427 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12428 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12429
12430 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12431 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12432 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12433 CPUMCTX2CORE(pCtx), GCPhys);
12434 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12435 if ( rcStrict2 == VINF_SUCCESS
12436 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12437 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12438 {
12439 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12440 | HM_CHANGED_GUEST_APIC_TPR);
12441 rcStrict2 = VINF_SUCCESS;
12442 }
12443 break;
12444 }
12445
12446 default:
12447 Log4Func(("uAccessType=%#x\n", uAccessType));
12448 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12449 break;
12450 }
12451
12452 if (rcStrict2 != VINF_SUCCESS)
12453 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12454 return rcStrict2;
12455}
12456
12457
12458/**
12459 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12460 * VM-exit.
12461 */
12462HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12463{
12464 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12465
12466 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12467 if (pVmxTransient->fWasGuestDebugStateActive)
12468 {
12469 AssertMsgFailed(("Unexpected MOV DRx exit\n"));
12470 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12471 }
12472
12473 if ( !pVCpu->hm.s.fSingleInstruction
12474 && !pVmxTransient->fWasHyperDebugStateActive)
12475 {
12476 Assert(!DBGFIsStepping(pVCpu));
12477 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12478
12479 /* Don't intercept MOV DRx any more. */
12480 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_PROC_CTLS_MOV_DR_EXIT;
12481 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12482 AssertRCReturn(rc, rc);
12483
12484 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12485 VMMRZCallRing3Disable(pVCpu);
12486 HM_DISABLE_PREEMPT(pVCpu);
12487
12488 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12489 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12490 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12491
12492 HM_RESTORE_PREEMPT();
12493 VMMRZCallRing3Enable(pVCpu);
12494
12495#ifdef VBOX_WITH_STATISTICS
12496 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12497 AssertRCReturn(rc, rc);
12498 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12500 else
12501 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12502#endif
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12504 return VINF_SUCCESS;
12505 }
12506
12507 /*
12508 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12509 * Update the segment registers and DR7 from the CPU.
12510 */
12511 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12512 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12513 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK | CPUMCTX_EXTRN_DR7);
12514 AssertRCReturn(rc, rc);
12515 Log4Func(("CS:RIP=%04x:%08RX64\n", pCtx->cs.Sel, pCtx->rip));
12516
12517 PVM pVM = pVCpu->CTX_SUFF(pVM);
12518 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12519 {
12520 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12521 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12522 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12523 if (RT_SUCCESS(rc))
12524 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12525 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12526 }
12527 else
12528 {
12529 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pCtx),
12530 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12531 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12532 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12533 }
12534
12535 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12536 if (RT_SUCCESS(rc))
12537 {
12538 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12539 AssertRCReturn(rc2, rc2);
12540 return VINF_SUCCESS;
12541 }
12542 return rc;
12543}
12544
12545
12546/**
12547 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12548 * Conditional VM-exit.
12549 */
12550HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12551{
12552 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12553 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12554
12555 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12556 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12557 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12558 {
12559 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12560 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12561 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12562 {
12563 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12564 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12565 }
12566 }
12567 else
12568 {
12569 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12570 rcStrict1 = VINF_SUCCESS;
12571 return rcStrict1;
12572 }
12573
12574 /*
12575 * Get sufficent state and update the exit history entry.
12576 */
12577 RTGCPHYS GCPhys;
12578 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12579 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12580 AssertRCReturn(rc, rc);
12581
12582 VBOXSTRICTRC rcStrict;
12583 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12584 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12585 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12586 if (!pExitRec)
12587 {
12588 /*
12589 * If we succeed, resume guest execution.
12590 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12591 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12592 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12593 * weird case. See @bugref{6043}.
12594 */
12595 PVM pVM = pVCpu->CTX_SUFF(pVM);
12596 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12597 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pCtx), GCPhys, UINT32_MAX);
12598 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12599 if ( rcStrict == VINF_SUCCESS
12600 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12601 || rcStrict == VERR_PAGE_NOT_PRESENT)
12602 {
12603 /* Successfully handled MMIO operation. */
12604 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
12605 | HM_CHANGED_GUEST_APIC_TPR);
12606 rcStrict = VINF_SUCCESS;
12607 }
12608 }
12609 else
12610 {
12611 /*
12612 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12613 */
12614 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12615 AssertRCReturn(rc2, rc2);
12616
12617 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12618 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12619
12620 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12621 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12622
12623 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12624 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12625 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12626 }
12627 return VBOXSTRICTRC_TODO(rcStrict);
12628}
12629
12630
12631/**
12632 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12633 * VM-exit.
12634 */
12635HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12636{
12637 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12638 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12639
12640 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12641 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pVmxTransient);
12642 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12643 {
12644 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12645 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12646 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12647 }
12648 else
12649 {
12650 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12651 rcStrict1 = VINF_SUCCESS;
12652 return rcStrict1;
12653 }
12654
12655 RTGCPHYS GCPhys;
12656 int rc = VMXReadVmcs64(VMX_VMCS64_RO_GUEST_PHYS_ADDR_FULL, &GCPhys);
12657 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12658 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12659 AssertRCReturn(rc, rc);
12660
12661 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12662 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12663
12664 RTGCUINT uErrorCode = 0;
12665 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12666 uErrorCode |= X86_TRAP_PF_ID;
12667 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12668 uErrorCode |= X86_TRAP_PF_RW;
12669 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12670 uErrorCode |= X86_TRAP_PF_P;
12671
12672 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12673
12674
12675 /* Handle the pagefault trap for the nested shadow table. */
12676 PVM pVM = pVCpu->CTX_SUFF(pVM);
12677 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12678
12679 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12680 uErrorCode, pCtx->cs.Sel, pCtx->rip));
12681
12682 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pCtx), GCPhys);
12683 TRPMResetTrap(pVCpu);
12684
12685 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12686 if ( rcStrict2 == VINF_SUCCESS
12687 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12688 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12689 {
12690 /* Successfully synced our nested page tables. */
12691 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12692 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
12693 return VINF_SUCCESS;
12694 }
12695
12696 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12697 return rcStrict2;
12698}
12699
12700/** @} */
12701
12702/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12703/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= VM-exit exception handlers =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
12704/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12705
12706/** @name VM-exit exception handlers.
12707 * @{
12708 */
12709
12710/**
12711 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12712 */
12713static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12714{
12715 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12716 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12717
12718 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12719 AssertRCReturn(rc, rc);
12720
12721 if (!(pVCpu->cpum.GstCtx.cr0 & X86_CR0_NE))
12722 {
12723 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12724 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12725
12726 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12727 * provides VM-exit instruction length. If this causes problem later,
12728 * disassemble the instruction like it's done on AMD-V. */
12729 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pVmxTransient);
12730 AssertRCReturn(rc2, rc2);
12731 return rc;
12732 }
12733
12734 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12735 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12736 return rc;
12737}
12738
12739
12740/**
12741 * VM-exit exception handler for \#BP (Breakpoint exception).
12742 */
12743static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12744{
12745 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12746 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12747
12748 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12749 AssertRCReturn(rc, rc);
12750
12751 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12752 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx));
12753 if (rc == VINF_EM_RAW_GUEST_TRAP)
12754 {
12755 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12756 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12757 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12758 AssertRCReturn(rc, rc);
12759
12760 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12761 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12762 }
12763
12764 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12765 return rc;
12766}
12767
12768
12769/**
12770 * VM-exit exception handler for \#AC (alignment check exception).
12771 */
12772static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12773{
12774 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12775
12776 /*
12777 * Re-inject it. We'll detect any nesting before getting here.
12778 */
12779 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12780 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12781 AssertRCReturn(rc, rc);
12782 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12783
12784 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12785 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12786 return VINF_SUCCESS;
12787}
12788
12789
12790/**
12791 * VM-exit exception handler for \#DB (Debug exception).
12792 */
12793static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12794{
12795 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12796 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12797
12798 /*
12799 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12800 * for processing.
12801 */
12802 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12803
12804 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12805 uint64_t uDR6 = X86_DR6_INIT_VAL;
12806 uDR6 |= ( pVmxTransient->uExitQualification
12807 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12808
12809 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12810 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12811 Log6Func(("rc=%Rrc\n", rc));
12812 if (rc == VINF_EM_RAW_GUEST_TRAP)
12813 {
12814 /*
12815 * The exception was for the guest. Update DR6, DR7.GD and
12816 * IA32_DEBUGCTL.LBR before forwarding it.
12817 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12818 */
12819 VMMRZCallRing3Disable(pVCpu);
12820 HM_DISABLE_PREEMPT(pVCpu);
12821
12822 pCtx->dr[6] &= ~X86_DR6_B_MASK;
12823 pCtx->dr[6] |= uDR6;
12824 if (CPUMIsGuestDebugStateActive(pVCpu))
12825 ASMSetDR6(pCtx->dr[6]);
12826
12827 HM_RESTORE_PREEMPT();
12828 VMMRZCallRing3Enable(pVCpu);
12829
12830 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12831 AssertRCReturn(rc, rc);
12832
12833 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12834 pCtx->dr[7] &= ~X86_DR7_GD;
12835
12836 /* Paranoia. */
12837 pCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12838 pCtx->dr[7] |= X86_DR7_RA1_MASK;
12839
12840 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pCtx->dr[7]);
12841 AssertRCReturn(rc, rc);
12842
12843 /*
12844 * Raise #DB in the guest.
12845 *
12846 * It is important to reflect exactly what the VM-exit gave us (preserving the
12847 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12848 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12849 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12850 *
12851 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12852 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12853 */
12854 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12855 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12856 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12857 AssertRCReturn(rc, rc);
12858 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12859 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12860 return VINF_SUCCESS;
12861 }
12862
12863 /*
12864 * Not a guest trap, must be a hypervisor related debug event then.
12865 * Update DR6 in case someone is interested in it.
12866 */
12867 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12868 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12869 CPUMSetHyperDR6(pVCpu, uDR6);
12870
12871 return rc;
12872}
12873
12874/**
12875 * VM-exit exception handler for \#GP (General-protection exception).
12876 *
12877 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12878 */
12879static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
12880{
12881 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
12882 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12883
12884 int rc;
12885 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
12886 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12887 { /* likely */ }
12888 else
12889 {
12890#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12891 Assert(pVCpu->hm.s.fUsingDebugLoop);
12892#endif
12893 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12894 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12895 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12896 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12897 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12898 AssertRCReturn(rc, rc);
12899 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pCtx->cs.Sel, pCtx->rip,
12900 pVmxTransient->uExitIntErrorCode, pCtx->cr0, CPUMGetGuestCPL(pVCpu), pCtx->tr.Sel));
12901 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12902 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12903 return rc;
12904 }
12905
12906 Assert(CPUMIsGuestInRealModeEx(pCtx));
12907 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12908
12909 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12910 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12911 AssertRCReturn(rc, rc);
12912
12913 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12914 uint32_t cbOp = 0;
12915 PVM pVM = pVCpu->CTX_SUFF(pVM);
12916 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12917 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12918 if (RT_SUCCESS(rc))
12919 {
12920 rc = VINF_SUCCESS;
12921 Assert(cbOp == pDis->cbInstr);
12922 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pCtx->cs.Sel, pCtx->rip));
12923 switch (pDis->pCurInstr->uOpcode)
12924 {
12925 case OP_CLI:
12926 {
12927 pCtx->eflags.Bits.u1IF = 0;
12928 pCtx->eflags.Bits.u1RF = 0;
12929 pCtx->rip += pDis->cbInstr;
12930 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12931 if ( !fDbgStepping
12932 && pCtx->eflags.Bits.u1TF)
12933 {
12934 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12935 AssertRCReturn(rc, rc);
12936 }
12937 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12938 break;
12939 }
12940
12941 case OP_STI:
12942 {
12943 bool fOldIF = pCtx->eflags.Bits.u1IF;
12944 pCtx->eflags.Bits.u1IF = 1;
12945 pCtx->eflags.Bits.u1RF = 0;
12946 pCtx->rip += pDis->cbInstr;
12947 if (!fOldIF)
12948 {
12949 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
12950 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12951 }
12952 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12953 if ( !fDbgStepping
12954 && pCtx->eflags.Bits.u1TF)
12955 {
12956 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
12957 AssertRCReturn(rc, rc);
12958 }
12959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12960 break;
12961 }
12962
12963 case OP_HLT:
12964 {
12965 rc = VINF_EM_HALT;
12966 pCtx->rip += pDis->cbInstr;
12967 pCtx->eflags.Bits.u1RF = 0;
12968 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12969 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12970 break;
12971 }
12972
12973 case OP_POPF:
12974 {
12975 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pCtx->cs.Sel, pCtx->rip));
12976 uint32_t cbParm;
12977 uint32_t uMask;
12978 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
12979 if (pDis->fPrefix & DISPREFIX_OPSIZE)
12980 {
12981 cbParm = 4;
12982 uMask = 0xffffffff;
12983 }
12984 else
12985 {
12986 cbParm = 2;
12987 uMask = 0xffff;
12988 }
12989
12990 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
12991 RTGCPTR GCPtrStack = 0;
12992 X86EFLAGS Eflags;
12993 Eflags.u32 = 0;
12994 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
12995 &GCPtrStack);
12996 if (RT_SUCCESS(rc))
12997 {
12998 Assert(sizeof(Eflags.u32) >= cbParm);
12999 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13000 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13001 }
13002 if (RT_FAILURE(rc))
13003 {
13004 rc = VERR_EM_INTERPRETER;
13005 break;
13006 }
13007 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pCtx->rsp, uMask, pCtx->rip));
13008 pCtx->eflags.u32 = (pCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13009 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13010 pCtx->esp += cbParm;
13011 pCtx->esp &= uMask;
13012 pCtx->rip += pDis->cbInstr;
13013 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13014 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13015 POPF restores EFLAGS.TF. */
13016 if ( !fDbgStepping
13017 && fGstStepping)
13018 {
13019 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13020 AssertRCReturn(rc, rc);
13021 }
13022 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13023 break;
13024 }
13025
13026 case OP_PUSHF:
13027 {
13028 uint32_t cbParm;
13029 uint32_t uMask;
13030 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13031 {
13032 cbParm = 4;
13033 uMask = 0xffffffff;
13034 }
13035 else
13036 {
13037 cbParm = 2;
13038 uMask = 0xffff;
13039 }
13040
13041 /* Get the stack pointer & push the contents of eflags onto the stack. */
13042 RTGCPTR GCPtrStack = 0;
13043 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), (pCtx->esp - cbParm) & uMask,
13044 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13045 if (RT_FAILURE(rc))
13046 {
13047 rc = VERR_EM_INTERPRETER;
13048 break;
13049 }
13050 X86EFLAGS Eflags = pCtx->eflags;
13051 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13052 Eflags.Bits.u1RF = 0;
13053 Eflags.Bits.u1VM = 0;
13054
13055 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13056 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13057 {
13058 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13059 rc = VERR_EM_INTERPRETER;
13060 break;
13061 }
13062 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13063 pCtx->esp -= cbParm;
13064 pCtx->esp &= uMask;
13065 pCtx->rip += pDis->cbInstr;
13066 pCtx->eflags.Bits.u1RF = 0;
13067 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS);
13068 if ( !fDbgStepping
13069 && pCtx->eflags.Bits.u1TF)
13070 {
13071 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13072 AssertRCReturn(rc, rc);
13073 }
13074 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13075 break;
13076 }
13077
13078 case OP_IRET:
13079 {
13080 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13081 * instruction reference. */
13082 RTGCPTR GCPtrStack = 0;
13083 uint32_t uMask = 0xffff;
13084 bool fGstStepping = RT_BOOL(pCtx->eflags.Bits.u1TF);
13085 uint16_t aIretFrame[3];
13086 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13087 {
13088 rc = VERR_EM_INTERPRETER;
13089 break;
13090 }
13091 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pCtx), pCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13092 &GCPtrStack);
13093 if (RT_SUCCESS(rc))
13094 {
13095 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13096 PGMACCESSORIGIN_HM));
13097 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13098 }
13099 if (RT_FAILURE(rc))
13100 {
13101 rc = VERR_EM_INTERPRETER;
13102 break;
13103 }
13104 pCtx->eip = 0;
13105 pCtx->ip = aIretFrame[0];
13106 pCtx->cs.Sel = aIretFrame[1];
13107 pCtx->cs.ValidSel = aIretFrame[1];
13108 pCtx->cs.u64Base = (uint64_t)pCtx->cs.Sel << 4;
13109 pCtx->eflags.u32 = (pCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13110 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13111 pCtx->sp += sizeof(aIretFrame);
13112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RSP | HM_CHANGED_GUEST_RFLAGS
13113 | HM_CHANGED_GUEST_CS);
13114 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13115 if ( !fDbgStepping
13116 && fGstStepping)
13117 {
13118 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu);
13119 AssertRCReturn(rc, rc);
13120 }
13121 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pCtx->cs.Sel, pCtx->ip));
13122 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13123 break;
13124 }
13125
13126 case OP_INT:
13127 {
13128 uint16_t uVector = pDis->Param1.uValue & 0xff;
13129 hmR0VmxSetPendingIntN(pVCpu, uVector, pDis->cbInstr);
13130 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13132 break;
13133 }
13134
13135 case OP_INTO:
13136 {
13137 if (pCtx->eflags.Bits.u1OF)
13138 {
13139 hmR0VmxSetPendingXcptOF(pVCpu, pDis->cbInstr);
13140 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13141 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13142 }
13143 else
13144 {
13145 pCtx->eflags.Bits.u1RF = 0;
13146 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13147 }
13148 break;
13149 }
13150
13151 default:
13152 {
13153 pCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13154 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pCtx), 0 /* pvFault */,
13155 EMCODETYPE_SUPERVISOR);
13156 rc = VBOXSTRICTRC_VAL(rc2);
13157 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13158 /** @todo We have to set pending-debug exceptions here when the guest is
13159 * single-stepping depending on the instruction that was interpreted. */
13160
13161 /*
13162 * HACK ALERT! Detect mode change and go to ring-3 to properly exit this
13163 * real mode emulation stuff.
13164 */
13165 if ( rc == VINF_SUCCESS
13166 && (pVCpu->cpum.GstCtx.cr0 & X86_CR0_PE))
13167 {
13168 Log4(("hmR0VmxExitXcptGP: mode changed -> VINF_EM_RESCHEDULE\n"));
13169 /** @todo Exit fRealOnV86Active here w/o dropping back to ring-3. */
13170 rc = VINF_EM_RESCHEDULE;
13171 }
13172
13173 Log4Func(("#GP rc=%Rrc\n", rc));
13174 break;
13175 }
13176 }
13177 }
13178 else
13179 rc = VERR_EM_INTERPRETER;
13180
13181 AssertMsg( rc == VINF_SUCCESS
13182 || rc == VERR_EM_INTERPRETER
13183 || rc == VINF_EM_HALT
13184 || rc == VINF_EM_RESCHEDULE
13185 , ("#GP Unexpected rc=%Rrc\n", rc));
13186 return rc;
13187}
13188
13189
13190/**
13191 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13192 * the exception reported in the VMX transient structure back into the VM.
13193 *
13194 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13195 * up-to-date.
13196 */
13197static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13198{
13199 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13200#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13201 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13202 ("uVector=%#x u32XcptBitmap=%#X32\n",
13203 VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13204#endif
13205
13206 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13207 hmR0VmxCheckExitDueToEventDelivery(). */
13208 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13209 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13210 AssertRCReturn(rc, rc);
13211 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13212
13213#ifdef DEBUG_ramshankar
13214 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS | CPUMCTX_EXTRN_RIP);
13215 uint8_t uVector = VMX_EXIT_INT_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13216 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13217#endif
13218
13219 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13220 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13221 return VINF_SUCCESS;
13222}
13223
13224
13225/**
13226 * VM-exit exception handler for \#PF (Page-fault exception).
13227 */
13228static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
13229{
13230 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS(pVCpu, pVmxTransient);
13231 PVM pVM = pVCpu->CTX_SUFF(pVM);
13232 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13233 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13234 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13235 AssertRCReturn(rc, rc);
13236
13237 if (!pVM->hm.s.fNestedPaging)
13238 { /* likely */ }
13239 else
13240 {
13241#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13242 Assert(pVCpu->hm.s.fUsingDebugLoop);
13243#endif
13244 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13245 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13246 {
13247 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13248 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13249 }
13250 else
13251 {
13252 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13253 hmR0VmxSetPendingXcptDF(pVCpu);
13254 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13255 }
13256 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13257 return rc;
13258 }
13259
13260 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13261 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13262 if (pVmxTransient->fVectoringPF)
13263 {
13264 Assert(pVCpu->hm.s.Event.fPending);
13265 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13266 }
13267
13268 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
13269 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13270 AssertRCReturn(rc, rc);
13271
13272 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13273 pCtx->cs.Sel, pCtx->rip, pVmxTransient->uExitIntErrorCode, pCtx->cr3));
13274
13275 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13276 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pCtx),
13277 (RTGCPTR)pVmxTransient->uExitQualification);
13278
13279 Log4Func(("#PF: rc=%Rrc\n", rc));
13280 if (rc == VINF_SUCCESS)
13281 {
13282 /*
13283 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13284 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13285 */
13286 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13287 TRPMResetTrap(pVCpu);
13288 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13289 return rc;
13290 }
13291
13292 if (rc == VINF_EM_RAW_GUEST_TRAP)
13293 {
13294 if (!pVmxTransient->fVectoringDoublePF)
13295 {
13296 /* It's a guest page fault and needs to be reflected to the guest. */
13297 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13298 TRPMResetTrap(pVCpu);
13299 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13300 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13301 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13302 }
13303 else
13304 {
13305 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13306 TRPMResetTrap(pVCpu);
13307 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13308 hmR0VmxSetPendingXcptDF(pVCpu);
13309 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13310 }
13311
13312 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13313 return VINF_SUCCESS;
13314 }
13315
13316 TRPMResetTrap(pVCpu);
13317 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13318 return rc;
13319}
13320
13321/** @} */
13322
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette