VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72878

Last change on this file since 72878 was 72878, checked in by vboxsync, 6 years ago

EM,HM,IEM: Replaced EMInterpretWrmsr with IEMExecDecodedWrmsr. [comment fixes]
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 577.0 KB
Line 
1/* $Id: HMVMXR0.cpp 72878 2018-07-04 14:30:02Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
152 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
335 * out-of-sync. Make sure to update the required
336 * fields before using them.
337 * @param pVmxTransient Pointer to the VMX-transient structure.
338 */
339#ifndef HMVMX_USE_FUNCTION_TABLE
340typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
341#else
342typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
343/** Pointer to VM-exit handler. */
344typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
345#endif
346
347/**
348 * VMX VM-exit handler, non-strict status code.
349 *
350 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
351 *
352 * @returns VBox status code, no informational status code returned.
353 * @param pVCpu The cross context virtual CPU structure.
354 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
355 * out-of-sync. Make sure to update the required
356 * fields before using them.
357 * @param pVmxTransient Pointer to the VMX-transient structure.
358 *
359 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
360 * use of that status code will be replaced with VINF_EM_SOMETHING
361 * later when switching over to IEM.
362 */
363#ifndef HMVMX_USE_FUNCTION_TABLE
364typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
365#else
366typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
367#endif
368
369
370/*********************************************************************************************************************************
371* Internal Functions *
372*********************************************************************************************************************************/
373static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush);
374static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr);
375static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
376static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
377static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
378 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
379#if HC_ARCH_BITS == 32
380static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
381#endif
382#ifndef HMVMX_USE_FUNCTION_TABLE
383DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
384# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
385# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
386#else
387# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
388# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
389#endif
390
391
392/** @name VM-exit handlers.
393 * @{
394 */
395static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
396static FNVMXEXITHANDLER hmR0VmxExitExtInt;
397static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
404static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
405static FNVMXEXITHANDLER hmR0VmxExitCpuid;
406static FNVMXEXITHANDLER hmR0VmxExitGetsec;
407static FNVMXEXITHANDLER hmR0VmxExitHlt;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
409static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
410static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
411static FNVMXEXITHANDLER hmR0VmxExitVmcall;
412static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423static FNVMXEXITHANDLER hmR0VmxExitMwait;
424static FNVMXEXITHANDLER hmR0VmxExitMtf;
425static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426static FNVMXEXITHANDLER hmR0VmxExitPause;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
432static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
433static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
434static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
435static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
437static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
438static FNVMXEXITHANDLER hmR0VmxExitRdrand;
439static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
440/** @} */
441
442static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
450
451
452/*********************************************************************************************************************************
453* Global Variables *
454*********************************************************************************************************************************/
455#ifdef HMVMX_USE_FUNCTION_TABLE
456
457/**
458 * VMX_EXIT dispatch table.
459 */
460static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
461{
462 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
463 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
464 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
465 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
466 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
467 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
468 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
469 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
470 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
471 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
472 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
473 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
474 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
475 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
476 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
477 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
478 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
479 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
480 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
481 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
482 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
483 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
484 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
485 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
486 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
487 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
488 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
489 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
490 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
491 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
492 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
493 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
494 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
495 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
496 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
497 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
498 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
499 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
500 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
501 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
502 /* 40 UNDEFINED */ hmR0VmxExitPause,
503 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
504 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
505 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
506 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
507 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
508 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
510 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
511 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
512 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
513 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
514 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
515 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
516 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
517 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
518 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
519 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
520 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
521 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
522 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
523 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
524 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
525 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
526 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
527};
528#endif /* HMVMX_USE_FUNCTION_TABLE */
529
530#ifdef VBOX_STRICT
531static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
532{
533 /* 0 */ "(Not Used)",
534 /* 1 */ "VMCALL executed in VMX root operation.",
535 /* 2 */ "VMCLEAR with invalid physical address.",
536 /* 3 */ "VMCLEAR with VMXON pointer.",
537 /* 4 */ "VMLAUNCH with non-clear VMCS.",
538 /* 5 */ "VMRESUME with non-launched VMCS.",
539 /* 6 */ "VMRESUME after VMXOFF",
540 /* 7 */ "VM-entry with invalid control fields.",
541 /* 8 */ "VM-entry with invalid host state fields.",
542 /* 9 */ "VMPTRLD with invalid physical address.",
543 /* 10 */ "VMPTRLD with VMXON pointer.",
544 /* 11 */ "VMPTRLD with incorrect revision identifier.",
545 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
546 /* 13 */ "VMWRITE to read-only VMCS component.",
547 /* 14 */ "(Not Used)",
548 /* 15 */ "VMXON executed in VMX root operation.",
549 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
550 /* 17 */ "VM-entry with non-launched executing VMCS.",
551 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
552 /* 19 */ "VMCALL with non-clear VMCS.",
553 /* 20 */ "VMCALL with invalid VM-exit control fields.",
554 /* 21 */ "(Not Used)",
555 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
556 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
557 /* 24 */ "VMCALL with invalid SMM-monitor features.",
558 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
559 /* 26 */ "VM-entry with events blocked by MOV SS.",
560 /* 27 */ "(Not Used)",
561 /* 28 */ "Invalid operand to INVEPT/INVVPID."
562};
563#endif /* VBOX_STRICT */
564
565
566
567/**
568 * Updates the VM's last error record.
569 *
570 * If there was a VMX instruction error, reads the error data from the VMCS and
571 * updates VCPU's last error record as well.
572 *
573 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
574 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575 * VERR_VMX_INVALID_VMCS_FIELD.
576 * @param rc The error code.
577 */
578static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
579{
580 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
581 || rc == VERR_VMX_UNABLE_TO_START_VM)
582 {
583 AssertPtrReturnVoid(pVCpu);
584 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
585 }
586 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
587}
588
589
590/**
591 * Reads the VM-entry interruption-information field from the VMCS into the VMX
592 * transient structure.
593 *
594 * @returns VBox status code.
595 * @param pVmxTransient Pointer to the VMX transient structure.
596 *
597 * @remarks No-long-jump zone!!!
598 */
599DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
600{
601 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
602 AssertRCReturn(rc, rc);
603 return VINF_SUCCESS;
604}
605
606#ifdef VBOX_STRICT
607/**
608 * Reads the VM-entry exception error code field from the VMCS into
609 * the VMX transient structure.
610 *
611 * @returns VBox status code.
612 * @param pVmxTransient Pointer to the VMX transient structure.
613 *
614 * @remarks No-long-jump zone!!!
615 */
616DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
617{
618 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
619 AssertRCReturn(rc, rc);
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-entry exception error code field from the VMCS into
626 * the VMX transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVmxTransient Pointer to the VMX transient structure.
630 *
631 * @remarks No-long-jump zone!!!
632 */
633DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
634{
635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
636 AssertRCReturn(rc, rc);
637 return VINF_SUCCESS;
638}
639#endif /* VBOX_STRICT */
640
641
642/**
643 * Reads the VM-exit interruption-information field from the VMCS into the VMX
644 * transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVmxTransient Pointer to the VMX transient structure.
648 */
649DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
650{
651 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
652 {
653 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
654 AssertRCReturn(rc,rc);
655 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
656 }
657 return VINF_SUCCESS;
658}
659
660
661/**
662 * Reads the VM-exit interruption error code from the VMCS into the VMX
663 * transient structure.
664 *
665 * @returns VBox status code.
666 * @param pVmxTransient Pointer to the VMX transient structure.
667 */
668DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
669{
670 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
671 {
672 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
673 AssertRCReturn(rc, rc);
674 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
675 }
676 return VINF_SUCCESS;
677}
678
679
680/**
681 * Reads the VM-exit instruction length field from the VMCS into the VMX
682 * transient structure.
683 *
684 * @returns VBox status code.
685 * @param pVmxTransient Pointer to the VMX transient structure.
686 */
687DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
688{
689 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
690 {
691 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
692 AssertRCReturn(rc, rc);
693 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
694 }
695 return VINF_SUCCESS;
696}
697
698
699/**
700 * Reads the VM-exit instruction-information field from the VMCS into
701 * the VMX transient structure.
702 *
703 * @returns VBox status code.
704 * @param pVmxTransient Pointer to the VMX transient structure.
705 */
706DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
707{
708 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
709 {
710 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
711 AssertRCReturn(rc, rc);
712 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
713 }
714 return VINF_SUCCESS;
715}
716
717
718/**
719 * Reads the exit code qualification from the VMCS into the VMX transient
720 * structure.
721 *
722 * @returns VBox status code.
723 * @param pVCpu The cross context virtual CPU structure of the
724 * calling EMT. (Required for the VMCS cache case.)
725 * @param pVmxTransient Pointer to the VMX transient structure.
726 */
727DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
728{
729 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
730 {
731 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
732 AssertRCReturn(rc, rc);
733 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
734 }
735 return VINF_SUCCESS;
736}
737
738
739/**
740 * Reads the IDT-vectoring information field from the VMCS into the VMX
741 * transient structure.
742 *
743 * @returns VBox status code.
744 * @param pVmxTransient Pointer to the VMX transient structure.
745 *
746 * @remarks No-long-jump zone!!!
747 */
748DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
749{
750 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
751 {
752 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
753 AssertRCReturn(rc, rc);
754 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
755 }
756 return VINF_SUCCESS;
757}
758
759
760/**
761 * Reads the IDT-vectoring error code from the VMCS into the VMX
762 * transient structure.
763 *
764 * @returns VBox status code.
765 * @param pVmxTransient Pointer to the VMX transient structure.
766 */
767DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
768{
769 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
770 {
771 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
772 AssertRCReturn(rc, rc);
773 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
774 }
775 return VINF_SUCCESS;
776}
777
778
779/**
780 * Enters VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 * @param pVM The cross context VM structure. Can be
784 * NULL, after a resume.
785 * @param HCPhysCpuPage Physical address of the VMXON region.
786 * @param pvCpuPage Pointer to the VMXON region.
787 */
788static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
789{
790 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
791 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
792 Assert(pvCpuPage);
793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
794
795 if (pVM)
796 {
797 /* Write the VMCS revision dword to the VMXON region. */
798 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
799 }
800
801 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
802 RTCCUINTREG fEFlags = ASMIntDisableFlags();
803
804 /* Enable the VMX bit in CR4 if necessary. */
805 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
806
807 /* Enter VMX root mode. */
808 int rc = VMXEnable(HCPhysCpuPage);
809 if (RT_FAILURE(rc))
810 {
811 if (!(uOldCr4 & X86_CR4_VMXE))
812 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
813
814 if (pVM)
815 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
816 }
817
818 /* Restore interrupts. */
819 ASMSetFlags(fEFlags);
820 return rc;
821}
822
823
824/**
825 * Exits VMX root mode operation on the current CPU.
826 *
827 * @returns VBox status code.
828 */
829static int hmR0VmxLeaveRootMode(void)
830{
831 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
832
833 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
834 RTCCUINTREG fEFlags = ASMIntDisableFlags();
835
836 /* If we're for some reason not in VMX root mode, then don't leave it. */
837 RTCCUINTREG uHostCR4 = ASMGetCR4();
838
839 int rc;
840 if (uHostCR4 & X86_CR4_VMXE)
841 {
842 /* Exit VMX root mode and clear the VMX bit in CR4. */
843 VMXDisable();
844 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
845 rc = VINF_SUCCESS;
846 }
847 else
848 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
849
850 /* Restore interrupts. */
851 ASMSetFlags(fEFlags);
852 return rc;
853}
854
855
856/**
857 * Allocates and maps one physically contiguous page. The allocated page is
858 * zero'd out. (Used by various VT-x structures).
859 *
860 * @returns IPRT status code.
861 * @param pMemObj Pointer to the ring-0 memory object.
862 * @param ppVirt Where to store the virtual address of the
863 * allocation.
864 * @param pHCPhys Where to store the physical address of the
865 * allocation.
866 */
867static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
868{
869 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
872
873 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
874 if (RT_FAILURE(rc))
875 return rc;
876 *ppVirt = RTR0MemObjAddress(*pMemObj);
877 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
878 ASMMemZero32(*ppVirt, PAGE_SIZE);
879 return VINF_SUCCESS;
880}
881
882
883/**
884 * Frees and unmaps an allocated physical page.
885 *
886 * @param pMemObj Pointer to the ring-0 memory object.
887 * @param ppVirt Where to re-initialize the virtual address of
888 * allocation as 0.
889 * @param pHCPhys Where to re-initialize the physical address of the
890 * allocation as 0.
891 */
892static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
893{
894 AssertPtr(pMemObj);
895 AssertPtr(ppVirt);
896 AssertPtr(pHCPhys);
897 if (*pMemObj != NIL_RTR0MEMOBJ)
898 {
899 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
900 AssertRC(rc);
901 *pMemObj = NIL_RTR0MEMOBJ;
902 *ppVirt = 0;
903 *pHCPhys = 0;
904 }
905}
906
907
908/**
909 * Worker function to free VT-x related structures.
910 *
911 * @returns IPRT status code.
912 * @param pVM The cross context VM structure.
913 */
914static void hmR0VmxStructsFree(PVM pVM)
915{
916 for (VMCPUID i = 0; i < pVM->cCpus; i++)
917 {
918 PVMCPU pVCpu = &pVM->aCpus[i];
919 AssertPtr(pVCpu);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
923
924 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
925 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
926
927 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 }
929
930 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
931#ifdef VBOX_WITH_CRASHDUMP_MAGIC
932 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
933#endif
934}
935
936
937/**
938 * Worker function to allocate VT-x related VM structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static int hmR0VmxStructsAlloc(PVM pVM)
944{
945 /*
946 * Initialize members up-front so we can cleanup properly on allocation failure.
947 */
948#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVM->hm.s.vmx.HCPhys##a_Name = 0;
952
953#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
954 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
955 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
956 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
957
958#ifdef VBOX_WITH_CRASHDUMP_MAGIC
959 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
960#endif
961 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
962
963 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
964 for (VMCPUID i = 0; i < pVM->cCpus; i++)
965 {
966 PVMCPU pVCpu = &pVM->aCpus[i];
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
971 }
972#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
973#undef VMXLOCAL_INIT_VM_MEMOBJ
974
975 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
976 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
977 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
978 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
979
980 /*
981 * Allocate all the VT-x structures.
982 */
983 int rc = VINF_SUCCESS;
984#ifdef VBOX_WITH_CRASHDUMP_MAGIC
985 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
989 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
990#endif
991
992 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
993 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
994 {
995 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
996 &pVM->hm.s.vmx.HCPhysApicAccess);
997 if (RT_FAILURE(rc))
998 goto cleanup;
999 }
1000
1001 /*
1002 * Initialize per-VCPU VT-x structures.
1003 */
1004 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1005 {
1006 PVMCPU pVCpu = &pVM->aCpus[i];
1007 AssertPtr(pVCpu);
1008
1009 /* Allocate the VM control structure (VMCS). */
1010 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1011 if (RT_FAILURE(rc))
1012 goto cleanup;
1013
1014 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1015 if ( PDMHasApic(pVM)
1016 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1017 {
1018 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1019 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1020 if (RT_FAILURE(rc))
1021 goto cleanup;
1022 }
1023
1024 /*
1025 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1026 * transparent accesses of specific MSRs.
1027 *
1028 * If the condition for enabling MSR bitmaps changes here, don't forget to
1029 * update HMAreMsrBitmapsAvailable().
1030 */
1031 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1032 {
1033 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1034 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1038 }
1039
1040 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1041 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044
1045 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1046 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1047 if (RT_FAILURE(rc))
1048 goto cleanup;
1049 }
1050
1051 return VINF_SUCCESS;
1052
1053cleanup:
1054 hmR0VmxStructsFree(pVM);
1055 return rc;
1056}
1057
1058
1059/**
1060 * Does global VT-x initialization (called during module initialization).
1061 *
1062 * @returns VBox status code.
1063 */
1064VMMR0DECL(int) VMXR0GlobalInit(void)
1065{
1066#ifdef HMVMX_USE_FUNCTION_TABLE
1067 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1068# ifdef VBOX_STRICT
1069 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1070 Assert(g_apfnVMExitHandlers[i]);
1071# endif
1072#endif
1073 return VINF_SUCCESS;
1074}
1075
1076
1077/**
1078 * Does global VT-x termination (called during module termination).
1079 */
1080VMMR0DECL(void) VMXR0GlobalTerm()
1081{
1082 /* Nothing to do currently. */
1083}
1084
1085
1086/**
1087 * Sets up and activates VT-x on the current CPU.
1088 *
1089 * @returns VBox status code.
1090 * @param pHostCpu Pointer to the global CPU info struct.
1091 * @param pVM The cross context VM structure. Can be
1092 * NULL after a host resume operation.
1093 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1094 * fEnabledByHost is @c true).
1095 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1096 * @a fEnabledByHost is @c true).
1097 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1098 * enable VT-x on the host.
1099 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1100 */
1101VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1102 void *pvMsrs)
1103{
1104 Assert(pHostCpu);
1105 Assert(pvMsrs);
1106 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1107
1108 /* Enable VT-x if it's not already enabled by the host. */
1109 if (!fEnabledByHost)
1110 {
1111 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1112 if (RT_FAILURE(rc))
1113 return rc;
1114 }
1115
1116 /*
1117 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1118 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1119 * invalidated when flushing by VPID.
1120 */
1121 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1122 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1123 {
1124 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXTLBFLUSHEPT_ALL_CONTEXTS);
1125 pHostCpu->fFlushAsidBeforeUse = false;
1126 }
1127 else
1128 pHostCpu->fFlushAsidBeforeUse = true;
1129
1130 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1131 ++pHostCpu->cTlbFlushes;
1132
1133 return VINF_SUCCESS;
1134}
1135
1136
1137/**
1138 * Deactivates VT-x on the current CPU.
1139 *
1140 * @returns VBox status code.
1141 * @param pHostCpu Pointer to the global CPU info struct.
1142 * @param pvCpuPage Pointer to the VMXON region.
1143 * @param HCPhysCpuPage Physical address of the VMXON region.
1144 *
1145 * @remarks This function should never be called when SUPR0EnableVTx() or
1146 * similar was used to enable VT-x on the host.
1147 */
1148VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1149{
1150 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1151
1152 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1153 return hmR0VmxLeaveRootMode();
1154}
1155
1156
1157/**
1158 * Sets the permission bits for the specified MSR in the MSR bitmap.
1159 *
1160 * @param pVCpu The cross context virtual CPU structure.
1161 * @param uMsr The MSR value.
1162 * @param enmRead Whether reading this MSR causes a VM-exit.
1163 * @param enmWrite Whether writing this MSR causes a VM-exit.
1164 */
1165static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1166{
1167 int32_t iBit;
1168 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170 /*
1171 * Layout:
1172 * 0x000 - 0x3ff - Low MSR read bits
1173 * 0x400 - 0x7ff - High MSR read bits
1174 * 0x800 - 0xbff - Low MSR write bits
1175 * 0xc00 - 0xfff - High MSR write bits
1176 */
1177 if (uMsr <= 0x00001FFF)
1178 iBit = uMsr;
1179 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1180 {
1181 iBit = uMsr - UINT32_C(0xC0000000);
1182 pbMsrBitmap += 0x400;
1183 }
1184 else
1185 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1186
1187 Assert(iBit <= 0x1fff);
1188 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1189 ASMBitSet(pbMsrBitmap, iBit);
1190 else
1191 ASMBitClear(pbMsrBitmap, iBit);
1192
1193 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1194 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1195 else
1196 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1197}
1198
1199
1200#ifdef VBOX_STRICT
1201/**
1202 * Gets the permission bits for the specified MSR in the MSR bitmap.
1203 *
1204 * @returns VBox status code.
1205 * @retval VINF_SUCCESS if the specified MSR is found.
1206 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1207 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1208 *
1209 * @param pVCpu The cross context virtual CPU structure.
1210 * @param uMsr The MSR.
1211 * @param penmRead Where to store the read permissions.
1212 * @param penmWrite Where to store the write permissions.
1213 */
1214static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1215{
1216 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1217 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1218 int32_t iBit;
1219 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1220
1221 /* See hmR0VmxSetMsrPermission() for the layout. */
1222 if (uMsr <= 0x00001FFF)
1223 iBit = uMsr;
1224 else if ( uMsr >= 0xC0000000
1225 && uMsr <= 0xC0001FFF)
1226 {
1227 iBit = (uMsr - 0xC0000000);
1228 pbMsrBitmap += 0x400;
1229 }
1230 else
1231 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1232
1233 Assert(iBit <= 0x1fff);
1234 if (ASMBitTest(pbMsrBitmap, iBit))
1235 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1236 else
1237 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1238
1239 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1240 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1241 else
1242 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1243 return VINF_SUCCESS;
1244}
1245#endif /* VBOX_STRICT */
1246
1247
1248/**
1249 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1250 * area.
1251 *
1252 * @returns VBox status code.
1253 * @param pVCpu The cross context virtual CPU structure.
1254 * @param cMsrs The number of MSRs.
1255 */
1256static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1257{
1258 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1259 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1260 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1261 {
1262 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1263 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1264 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1265 }
1266
1267 /* Update number of guest MSRs to load/store across the world-switch. */
1268 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1269 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1270
1271 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1273 AssertRCReturn(rc, rc);
1274
1275 /* Update the VCPU's copy of the MSR count. */
1276 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1277
1278 return VINF_SUCCESS;
1279}
1280
1281
1282/**
1283 * Adds a new (or updates the value of an existing) guest/host MSR
1284 * pair to be swapped during the world-switch as part of the
1285 * auto-load/store MSR area in the VMCS.
1286 *
1287 * @returns VBox status code.
1288 * @param pVCpu The cross context virtual CPU structure.
1289 * @param uMsr The MSR.
1290 * @param uGuestMsrValue Value of the guest MSR.
1291 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1292 * necessary.
1293 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1294 * its value was updated. Optional, can be NULL.
1295 */
1296static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1297 bool *pfAddedAndUpdated)
1298{
1299 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1300 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1301 uint32_t i;
1302 for (i = 0; i < cMsrs; i++)
1303 {
1304 if (pGuestMsr->u32Msr == uMsr)
1305 break;
1306 pGuestMsr++;
1307 }
1308
1309 bool fAdded = false;
1310 if (i == cMsrs)
1311 {
1312 ++cMsrs;
1313 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1314 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1315
1316 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1317 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1318 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1319
1320 fAdded = true;
1321 }
1322
1323 /* Update the MSR values in the auto-load/store MSR area. */
1324 pGuestMsr->u32Msr = uMsr;
1325 pGuestMsr->u64Value = uGuestMsrValue;
1326
1327 /* Create/update the MSR slot in the host MSR area. */
1328 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1329 pHostMsr += i;
1330 pHostMsr->u32Msr = uMsr;
1331
1332 /*
1333 * Update the host MSR only when requested by the caller AND when we're
1334 * adding it to the auto-load/store area. Otherwise, it would have been
1335 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1336 */
1337 bool fUpdatedMsrValue = false;
1338 if ( fAdded
1339 && fUpdateHostMsr)
1340 {
1341 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1342 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1343 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1344 fUpdatedMsrValue = true;
1345 }
1346
1347 if (pfAddedAndUpdated)
1348 *pfAddedAndUpdated = fUpdatedMsrValue;
1349 return VINF_SUCCESS;
1350}
1351
1352
1353/**
1354 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1355 * auto-load/store MSR area in the VMCS.
1356 *
1357 * @returns VBox status code.
1358 * @param pVCpu The cross context virtual CPU structure.
1359 * @param uMsr The MSR.
1360 */
1361static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1362{
1363 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1364 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1365 for (uint32_t i = 0; i < cMsrs; i++)
1366 {
1367 /* Find the MSR. */
1368 if (pGuestMsr->u32Msr == uMsr)
1369 {
1370 /* If it's the last MSR, simply reduce the count. */
1371 if (i == cMsrs - 1)
1372 {
1373 --cMsrs;
1374 break;
1375 }
1376
1377 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1378 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1379 pLastGuestMsr += cMsrs - 1;
1380 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1381 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1382
1383 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1385 pLastHostMsr += cMsrs - 1;
1386 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1387 pHostMsr->u64Value = pLastHostMsr->u64Value;
1388 --cMsrs;
1389 break;
1390 }
1391 pGuestMsr++;
1392 }
1393
1394 /* Update the VMCS if the count changed (meaning the MSR was found). */
1395 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1396 {
1397 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1398 AssertRCReturn(rc, rc);
1399
1400 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1401 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1402 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1403
1404 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1405 return VINF_SUCCESS;
1406 }
1407
1408 return VERR_NOT_FOUND;
1409}
1410
1411
1412/**
1413 * Checks if the specified guest MSR is part of the auto-load/store area in
1414 * the VMCS.
1415 *
1416 * @returns true if found, false otherwise.
1417 * @param pVCpu The cross context virtual CPU structure.
1418 * @param uMsr The MSR to find.
1419 */
1420static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1421{
1422 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1423 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1424
1425 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1426 {
1427 if (pGuestMsr->u32Msr == uMsr)
1428 return true;
1429 }
1430 return false;
1431}
1432
1433
1434/**
1435 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1436 *
1437 * @param pVCpu The cross context virtual CPU structure.
1438 *
1439 * @remarks No-long-jump zone!!!
1440 */
1441static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1442{
1443 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1444 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1445 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1446 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1447
1448 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1449 {
1450 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1451
1452 /*
1453 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1454 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1455 */
1456 if (pHostMsr->u32Msr == MSR_K6_EFER)
1457 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1458 else
1459 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1460 }
1461
1462 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1463}
1464
1465
1466/**
1467 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1468 * perform lazy restoration of the host MSRs while leaving VT-x.
1469 *
1470 * @param pVCpu The cross context virtual CPU structure.
1471 *
1472 * @remarks No-long-jump zone!!!
1473 */
1474static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1475{
1476 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1477
1478 /*
1479 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1480 */
1481 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1482 {
1483 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1484#if HC_ARCH_BITS == 64
1485 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1486 {
1487 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1488 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1489 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1490 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1491 }
1492#endif
1493 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1494 }
1495}
1496
1497
1498/**
1499 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1500 * lazily while leaving VT-x.
1501 *
1502 * @returns true if it does, false otherwise.
1503 * @param pVCpu The cross context virtual CPU structure.
1504 * @param uMsr The MSR to check.
1505 */
1506static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1507{
1508 NOREF(pVCpu);
1509#if HC_ARCH_BITS == 64
1510 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1511 {
1512 switch (uMsr)
1513 {
1514 case MSR_K8_LSTAR:
1515 case MSR_K6_STAR:
1516 case MSR_K8_SF_MASK:
1517 case MSR_K8_KERNEL_GS_BASE:
1518 return true;
1519 }
1520 }
1521#else
1522 RT_NOREF(pVCpu, uMsr);
1523#endif
1524 return false;
1525}
1526
1527
1528/**
1529 * Loads a set of guests MSRs to allow read/passthru to the guest.
1530 *
1531 * The name of this function is slightly confusing. This function does NOT
1532 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1533 * common prefix for functions dealing with "lazy restoration" of the shared
1534 * MSRs.
1535 *
1536 * @param pVCpu The cross context virtual CPU structure.
1537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1538 * out-of-sync. Make sure to update the required fields
1539 * before using them.
1540 *
1541 * @remarks No-long-jump zone!!!
1542 */
1543static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1544{
1545 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1546 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1547
1548 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1549#if HC_ARCH_BITS == 64
1550 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1551 {
1552 /*
1553 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1554 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1555 * we can skip a few MSR writes.
1556 *
1557 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1558 * guest MSR values in the guest-CPU context might be different to what's currently
1559 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1560 * CPU, see @bugref{8728}.
1561 */
1562 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1564 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1565 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1566 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1567 {
1568#ifdef VBOX_STRICT
1569 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1570 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1571 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1572 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1573#endif
1574 }
1575 else
1576 {
1577 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1578 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1579 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1580 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1581 }
1582 }
1583#else
1584 RT_NOREF(pMixedCtx);
1585#endif
1586 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1587}
1588
1589
1590/**
1591 * Performs lazy restoration of the set of host MSRs if they were previously
1592 * loaded with guest MSR values.
1593 *
1594 * @param pVCpu The cross context virtual CPU structure.
1595 *
1596 * @remarks No-long-jump zone!!!
1597 * @remarks The guest MSRs should have been saved back into the guest-CPU
1598 * context by hmR0VmxImportGuestState()!!!
1599 */
1600static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1601{
1602 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1604
1605 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1606 {
1607 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1608#if HC_ARCH_BITS == 64
1609 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1610 {
1611 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1612 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1613 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1614 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1615 }
1616#endif
1617 }
1618 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1619}
1620
1621
1622/**
1623 * Verifies that our cached values of the VMCS fields are all consistent with
1624 * what's actually present in the VMCS.
1625 *
1626 * @returns VBox status code.
1627 * @retval VINF_SUCCESS if all our caches match their respective VMCS fields.
1628 * @retval VERR_VMX_VMCS_FIELD_CACHE_INVALID if a cache field doesn't match the
1629 * VMCS content. HMCPU error-field is
1630 * updated, see VMX_VCI_XXX.
1631 * @param pVCpu The cross context virtual CPU structure.
1632 */
1633static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1634{
1635 uint32_t u32Val;
1636 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32EntryCtls == u32Val,
1639 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1640 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_ENTRY,
1641 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1642
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ExitCtls == u32Val,
1646 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1647 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_EXIT,
1648 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32PinCtls == u32Val,
1653 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1654 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PIN_EXEC,
1655 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1656
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1661 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC,
1662 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1663
1664 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1665 {
1666 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1667 AssertRCReturn(rc, rc);
1668 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1669 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1670 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_PROC_EXEC2,
1671 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1672 }
1673
1674 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val);
1675 AssertRCReturn(rc, rc);
1676 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u32XcptBitmap == u32Val,
1677 ("Cache=%#RX32 VMCS=%#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap, u32Val),
1678 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_XCPT_BITMAP,
1679 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1680
1681 uint64_t u64Val;
1682 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, &u64Val);
1683 AssertRCReturn(rc, rc);
1684 AssertMsgReturnStmt(pVCpu->hm.s.vmx.u64TscOffset == u64Val,
1685 ("Cache=%#RX64 VMCS=%#RX64\n", pVCpu->hm.s.vmx.u64TscOffset, u64Val),
1686 pVCpu->hm.s.u32HMError = VMX_VCI_CTRL_TSC_OFFSET,
1687 VERR_VMX_VMCS_FIELD_CACHE_INVALID);
1688
1689 return VINF_SUCCESS;
1690}
1691
1692
1693#ifdef VBOX_STRICT
1694/**
1695 * Verifies that our cached host EFER value has not changed
1696 * since we cached it.
1697 *
1698 * @param pVCpu The cross context virtual CPU structure.
1699 */
1700static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1701{
1702 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1703
1704 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1705 {
1706 uint64_t u64Val;
1707 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1708 AssertRC(rc);
1709
1710 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1711 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1712 }
1713}
1714
1715
1716/**
1717 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1718 * VMCS are correct.
1719 *
1720 * @param pVCpu The cross context virtual CPU structure.
1721 */
1722static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1723{
1724 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1725
1726 /* Verify MSR counts in the VMCS are what we think it should be. */
1727 uint32_t cMsrs;
1728 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1729 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1730
1731 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1732 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1733
1734 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1735 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1736
1737 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1738 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1739 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1740 {
1741 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1742 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1743 pGuestMsr->u32Msr, cMsrs));
1744
1745 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1746 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1747 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1748
1749 /* Verify that the permissions are as expected in the MSR bitmap. */
1750 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1751 {
1752 VMXMSREXITREAD enmRead;
1753 VMXMSREXITWRITE enmWrite;
1754 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1755 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1756 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1757 {
1758 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1759 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1760 }
1761 else
1762 {
1763 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1764 pGuestMsr->u32Msr, cMsrs));
1765 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1766 pGuestMsr->u32Msr, cMsrs));
1767 }
1768 }
1769 }
1770}
1771#endif /* VBOX_STRICT */
1772
1773
1774/**
1775 * Flushes the TLB using EPT.
1776 *
1777 * @returns VBox status code.
1778 * @param pVCpu The cross context virtual CPU structure of the calling
1779 * EMT. Can be NULL depending on @a enmTlbFlush.
1780 * @param enmTlbFlush Type of flush.
1781 *
1782 * @remarks Caller is responsible for making sure this function is called only
1783 * when NestedPaging is supported and providing @a enmTlbFlush that is
1784 * supported by the CPU.
1785 * @remarks Can be called with interrupts disabled.
1786 */
1787static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXTLBFLUSHEPT enmTlbFlush)
1788{
1789 uint64_t au64Descriptor[2];
1790 if (enmTlbFlush == VMXTLBFLUSHEPT_ALL_CONTEXTS)
1791 au64Descriptor[0] = 0;
1792 else
1793 {
1794 Assert(pVCpu);
1795 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1796 }
1797 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1798
1799 int rc = VMXR0InvEPT(enmTlbFlush, &au64Descriptor[0]);
1800 AssertMsg(rc == VINF_SUCCESS,
1801 ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0, rc));
1802
1803 if ( RT_SUCCESS(rc)
1804 && pVCpu)
1805 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1806}
1807
1808
1809/**
1810 * Flushes the TLB using VPID.
1811 *
1812 * @returns VBox status code.
1813 * @param pVCpu The cross context virtual CPU structure of the calling
1814 * EMT. Can be NULL depending on @a enmTlbFlush.
1815 * @param enmTlbFlush Type of flush.
1816 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1817 * on @a enmTlbFlush).
1818 *
1819 * @remarks Can be called with interrupts disabled.
1820 */
1821static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXTLBFLUSHVPID enmTlbFlush, RTGCPTR GCPtr)
1822{
1823 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1824
1825 uint64_t au64Descriptor[2];
1826 if (enmTlbFlush == VMXTLBFLUSHVPID_ALL_CONTEXTS)
1827 {
1828 au64Descriptor[0] = 0;
1829 au64Descriptor[1] = 0;
1830 }
1831 else
1832 {
1833 AssertPtr(pVCpu);
1834 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1835 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1836 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1837 au64Descriptor[1] = GCPtr;
1838 }
1839
1840 int rc = VMXR0InvVPID(enmTlbFlush, &au64Descriptor[0]);
1841 AssertMsg(rc == VINF_SUCCESS,
1842 ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmTlbFlush, pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1843
1844 if ( RT_SUCCESS(rc)
1845 && pVCpu)
1846 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1847 NOREF(rc);
1848}
1849
1850
1851/**
1852 * Invalidates a guest page by guest virtual address. Only relevant for
1853 * EPT/VPID, otherwise there is nothing really to invalidate.
1854 *
1855 * @returns VBox status code.
1856 * @param pVCpu The cross context virtual CPU structure.
1857 * @param GCVirt Guest virtual address of the page to invalidate.
1858 */
1859VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1860{
1861 AssertPtr(pVCpu);
1862 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1863
1864 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1865 if (!fFlushPending)
1866 {
1867 /*
1868 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1869 * the EPT case. See @bugref{6043} and @bugref{6177}.
1870 *
1871 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1872 * as this function maybe called in a loop with individual addresses.
1873 */
1874 PVM pVM = pVCpu->CTX_SUFF(pVM);
1875 if (pVM->hm.s.vmx.fVpid)
1876 {
1877 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1878
1879#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1880 /*
1881 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1882 * where executing INVVPID outside 64-bit mode does not flush translations of
1883 * 64-bit linear addresses, see @bugref{6208#c72}.
1884 */
1885 if (RT_HI_U32(GCVirt))
1886 fVpidFlush = false;
1887#endif
1888
1889 if (fVpidFlush)
1890 {
1891 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_INDIV_ADDR, GCVirt);
1892 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1893 }
1894 else
1895 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1896 }
1897 else if (pVM->hm.s.fNestedPaging)
1898 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1899 }
1900
1901 return VINF_SUCCESS;
1902}
1903
1904
1905/**
1906 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1907 * case where neither EPT nor VPID is supported by the CPU.
1908 *
1909 * @param pVCpu The cross context virtual CPU structure.
1910 * @param pCpu Pointer to the global HM struct.
1911 *
1912 * @remarks Called with interrupts disabled.
1913 */
1914static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915{
1916 AssertPtr(pVCpu);
1917 AssertPtr(pCpu);
1918
1919 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1920
1921 Assert(pCpu->idCpu != NIL_RTCPUID);
1922 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1923 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1924 pVCpu->hm.s.fForceTLBFlush = false;
1925 return;
1926}
1927
1928
1929/**
1930 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1931 *
1932 * @param pVCpu The cross context virtual CPU structure.
1933 * @param pCpu Pointer to the global HM CPU struct.
1934 *
1935 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1936 * nomenclature. The reason is, to avoid confusion in compare statements
1937 * since the host-CPU copies are named "ASID".
1938 *
1939 * @remarks Called with interrupts disabled.
1940 */
1941static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1942{
1943#ifdef VBOX_WITH_STATISTICS
1944 bool fTlbFlushed = false;
1945# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1946# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1947 if (!fTlbFlushed) \
1948 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1949 } while (0)
1950#else
1951# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1952# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1953#endif
1954
1955 AssertPtr(pCpu);
1956 AssertPtr(pVCpu);
1957 Assert(pCpu->idCpu != NIL_RTCPUID);
1958
1959 PVM pVM = pVCpu->CTX_SUFF(pVM);
1960 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1961 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1962 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1963
1964 /*
1965 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1966 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1967 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1968 * cannot reuse the current ASID anymore.
1969 */
1970 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1971 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1972 {
1973 ++pCpu->uCurrentAsid;
1974 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1975 {
1976 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1977 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1978 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1979 }
1980
1981 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1982 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1983 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1984
1985 /*
1986 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1987 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1988 */
1989 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
1990 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1991 HMVMX_SET_TAGGED_TLB_FLUSHED();
1992 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1993 }
1994
1995 /* Check for explicit TLB flushes. */
1996 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1997 {
1998 /*
1999 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
2000 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
2001 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
2002 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
2003 * mappings, see @bugref{6568}.
2004 *
2005 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
2006 */
2007 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmTlbFlushEpt);
2008 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2009 HMVMX_SET_TAGGED_TLB_FLUSHED();
2010 }
2011
2012 pVCpu->hm.s.fForceTLBFlush = false;
2013 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
2014
2015 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
2016 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
2017 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2018 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2019 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2020 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2021 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2022 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2023 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2024
2025 /* Update VMCS with the VPID. */
2026 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2027 AssertRC(rc);
2028
2029#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2030}
2031
2032
2033/**
2034 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2035 *
2036 * @returns VBox status code.
2037 * @param pVCpu The cross context virtual CPU structure.
2038 * @param pCpu Pointer to the global HM CPU struct.
2039 *
2040 * @remarks Called with interrupts disabled.
2041 */
2042static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2043{
2044 AssertPtr(pVCpu);
2045 AssertPtr(pCpu);
2046 Assert(pCpu->idCpu != NIL_RTCPUID);
2047 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2048 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2049
2050 /*
2051 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2052 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2053 */
2054 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2055 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2056 {
2057 pVCpu->hm.s.fForceTLBFlush = true;
2058 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2059 }
2060
2061 /* Check for explicit TLB flushes. */
2062 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2063 {
2064 pVCpu->hm.s.fForceTLBFlush = true;
2065 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2066 }
2067
2068 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2069 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2070
2071 if (pVCpu->hm.s.fForceTLBFlush)
2072 {
2073 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmTlbFlushEpt);
2074 pVCpu->hm.s.fForceTLBFlush = false;
2075 }
2076}
2077
2078
2079/**
2080 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2081 *
2082 * @returns VBox status code.
2083 * @param pVCpu The cross context virtual CPU structure.
2084 * @param pCpu Pointer to the global HM CPU struct.
2085 *
2086 * @remarks Called with interrupts disabled.
2087 */
2088static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2089{
2090 AssertPtr(pVCpu);
2091 AssertPtr(pCpu);
2092 Assert(pCpu->idCpu != NIL_RTCPUID);
2093 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2094 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2095
2096 /*
2097 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2098 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2099 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2100 * cannot reuse the current ASID anymore.
2101 */
2102 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2103 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2104 {
2105 pVCpu->hm.s.fForceTLBFlush = true;
2106 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2107 }
2108
2109 /* Check for explicit TLB flushes. */
2110 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2111 {
2112 /*
2113 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2114 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2115 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2116 * include fExplicitFlush's too) - an obscure corner case.
2117 */
2118 pVCpu->hm.s.fForceTLBFlush = true;
2119 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2120 }
2121
2122 PVM pVM = pVCpu->CTX_SUFF(pVM);
2123 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2124 if (pVCpu->hm.s.fForceTLBFlush)
2125 {
2126 ++pCpu->uCurrentAsid;
2127 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2128 {
2129 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2130 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2131 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2132 }
2133
2134 pVCpu->hm.s.fForceTLBFlush = false;
2135 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2136 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2137 if (pCpu->fFlushAsidBeforeUse)
2138 {
2139 if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_SINGLE_CONTEXT)
2140 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2141 else if (pVM->hm.s.vmx.enmTlbFlushVpid == VMXTLBFLUSHVPID_ALL_CONTEXTS)
2142 {
2143 hmR0VmxFlushVpid(pVCpu, VMXTLBFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2144 pCpu->fFlushAsidBeforeUse = false;
2145 }
2146 else
2147 {
2148 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2149 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2150 }
2151 }
2152 }
2153
2154 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2155 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2156 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2157 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2158 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2159 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2160 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2161
2162 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2163 AssertRC(rc);
2164}
2165
2166
2167/**
2168 * Flushes the guest TLB entry based on CPU capabilities.
2169 *
2170 * @param pVCpu The cross context virtual CPU structure.
2171 * @param pCpu Pointer to the global HM CPU struct.
2172 */
2173DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2174{
2175#ifdef HMVMX_ALWAYS_FLUSH_TLB
2176 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2177#endif
2178 PVM pVM = pVCpu->CTX_SUFF(pVM);
2179 switch (pVM->hm.s.vmx.enmTlbFlushType)
2180 {
2181 case VMXTLBFLUSHTYPE_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2182 case VMXTLBFLUSHTYPE_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2183 case VMXTLBFLUSHTYPE_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2184 case VMXTLBFLUSHTYPE_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2185 default:
2186 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2187 break;
2188 }
2189 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2190}
2191
2192
2193/**
2194 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2195 * TLB entries from the host TLB before VM-entry.
2196 *
2197 * @returns VBox status code.
2198 * @param pVM The cross context VM structure.
2199 */
2200static int hmR0VmxSetupTaggedTlb(PVM pVM)
2201{
2202 /*
2203 * Determine optimal flush type for Nested Paging.
2204 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2205 * guest execution (see hmR3InitFinalizeR0()).
2206 */
2207 if (pVM->hm.s.fNestedPaging)
2208 {
2209 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2210 {
2211 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2212 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_SINGLE_CONTEXT;
2213 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2214 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_ALL_CONTEXTS;
2215 else
2216 {
2217 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2218 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2219 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2220 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2221 }
2222
2223 /* Make sure the write-back cacheable memory type for EPT is supported. */
2224 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2225 {
2226 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2227 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2228 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2229 }
2230
2231 /* EPT requires a page-walk length of 4. */
2232 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2233 {
2234 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2235 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2236 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2237 }
2238 }
2239 else
2240 {
2241 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2242 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NOT_SUPPORTED;
2243 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2244 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2245 }
2246 }
2247
2248 /*
2249 * Determine optimal flush type for VPID.
2250 */
2251 if (pVM->hm.s.vmx.fVpid)
2252 {
2253 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2254 {
2255 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2256 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_SINGLE_CONTEXT;
2257 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2258 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_ALL_CONTEXTS;
2259 else
2260 {
2261 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2262 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2263 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2264 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2265 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2266 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2267 pVM->hm.s.vmx.fVpid = false;
2268 }
2269 }
2270 else
2271 {
2272 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2273 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2274 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NOT_SUPPORTED;
2275 pVM->hm.s.vmx.fVpid = false;
2276 }
2277 }
2278
2279 /*
2280 * Setup the handler for flushing tagged-TLBs.
2281 */
2282 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2283 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT_VPID;
2284 else if (pVM->hm.s.fNestedPaging)
2285 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_EPT;
2286 else if (pVM->hm.s.vmx.fVpid)
2287 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_VPID;
2288 else
2289 pVM->hm.s.vmx.enmTlbFlushType = VMXTLBFLUSHTYPE_NONE;
2290 return VINF_SUCCESS;
2291}
2292
2293
2294/**
2295 * Sets up pin-based VM-execution controls in the VMCS.
2296 *
2297 * @returns VBox status code.
2298 * @param pVCpu The cross context virtual CPU structure.
2299 *
2300 * @remarks We don't really care about optimizing vmwrites here as it's done only
2301 * once per VM and hence we don't care about VMCS-field cache comparisons.
2302 */
2303static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2304{
2305 PVM pVM = pVCpu->CTX_SUFF(pVM);
2306 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2307 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2308
2309 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2310 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2311
2312 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2313 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2314
2315 /* Enable the VMX preemption timer. */
2316 if (pVM->hm.s.vmx.fUsePreemptTimer)
2317 {
2318 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2319 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2320 }
2321
2322#if 0
2323 /* Enable posted-interrupt processing. */
2324 if (pVM->hm.s.fPostedIntrs)
2325 {
2326 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2327 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2328 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2329 }
2330#endif
2331
2332 if ((fVal & fZap) != fVal)
2333 {
2334 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2335 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2336 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2337 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2338 }
2339
2340 /* Commit it to the VMCS and update our cache. */
2341 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2342 AssertRCReturn(rc, rc);
2343 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2344
2345 return VINF_SUCCESS;
2346}
2347
2348
2349/**
2350 * Sets up secondary processor-based VM-execution controls in the VMCS.
2351 *
2352 * @returns VBox status code.
2353 * @param pVCpu The cross context virtual CPU structure.
2354 *
2355 * @remarks We don't really care about optimizing vmwrites here as it's done only
2356 * once per VM and hence we don't care about VMCS-field cache comparisons.
2357 */
2358static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2359{
2360 PVM pVM = pVCpu->CTX_SUFF(pVM);
2361 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2362 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2363
2364 /* WBINVD causes a VM-exit. */
2365 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2366 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2367
2368 /* Enable EPT (aka nested-paging). */
2369 if (pVM->hm.s.fNestedPaging)
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2371
2372 /*
2373 * Enable the INVPCID instruction if supported by the hardware and we expose
2374 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2375 */
2376 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2377 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2378 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2379
2380 /* Enable VPID. */
2381 if (pVM->hm.s.vmx.fVpid)
2382 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2383
2384 /* Enable Unrestricted guest execution. */
2385 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2386 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2387
2388#if 0
2389 if (pVM->hm.s.fVirtApicRegs)
2390 {
2391 /* Enable APIC-register virtualization. */
2392 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2393 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2394
2395 /* Enable virtual-interrupt delivery. */
2396 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2397 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2398 }
2399#endif
2400
2401 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2402 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2403 * done dynamically. */
2404 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2405 {
2406 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2407 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2408 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2409 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2410 AssertRCReturn(rc, rc);
2411 }
2412
2413 /* Enable RDTSCP. */
2414 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2415 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2416
2417 /* Enable Pause-Loop exiting. */
2418 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2419 && pVM->hm.s.vmx.cPleGapTicks
2420 && pVM->hm.s.vmx.cPleWindowTicks)
2421 {
2422 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2423
2424 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2425 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2426 AssertRCReturn(rc, rc);
2427 }
2428
2429 if ((fVal & fZap) != fVal)
2430 {
2431 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2432 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2433 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2434 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2435 }
2436
2437 /* Commit it to the VMCS and update our cache. */
2438 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2439 AssertRCReturn(rc, rc);
2440 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2441
2442 return VINF_SUCCESS;
2443}
2444
2445
2446/**
2447 * Sets up processor-based VM-execution controls in the VMCS.
2448 *
2449 * @returns VBox status code.
2450 * @param pVCpu The cross context virtual CPU structure.
2451 *
2452 * @remarks We don't really care about optimizing vmwrites here as it's done only
2453 * once per VM and hence we don't care about VMCS-field cache comparisons.
2454 */
2455static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2456{
2457 PVM pVM = pVCpu->CTX_SUFF(pVM);
2458 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2459 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2460
2461 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2462 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2463 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2464 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2465 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2466 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2467 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2468
2469 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2470 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2471 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2472 {
2473 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2474 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2475 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2476 }
2477
2478 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2479 if (!pVM->hm.s.fNestedPaging)
2480 {
2481 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2482 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2483 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2484 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2485 }
2486
2487 /* Use TPR shadowing if supported by the CPU. */
2488 if ( PDMHasApic(pVM)
2489 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2490 {
2491 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2492 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2493 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2494 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2495 AssertRCReturn(rc, rc);
2496
2497 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2498 /* CR8 writes cause a VM-exit based on TPR threshold. */
2499 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2500 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2501 }
2502 else
2503 {
2504 /*
2505 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2506 * Set this control only for 64-bit guests.
2507 */
2508 if (pVM->hm.s.fAllow64BitGuests)
2509 {
2510 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2511 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2512 }
2513 }
2514
2515 /* Use MSR-bitmaps if supported by the CPU. */
2516 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2517 {
2518 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2519
2520 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2521 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2522 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2523 AssertRCReturn(rc, rc);
2524
2525 /*
2526 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2527 * automatically using dedicated fields in the VMCS.
2528 */
2529 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2530 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2531 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2532 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2533 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2534#if HC_ARCH_BITS == 64
2535 /*
2536 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2537 */
2538 if (pVM->hm.s.fAllow64BitGuests)
2539 {
2540 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2541 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2542 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2543 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2544 }
2545#endif
2546 /*
2547 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2548 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2549 */
2550 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2551 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2552
2553 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2554 }
2555
2556 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2557 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2558 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2559
2560 if ((fVal & fZap) != fVal)
2561 {
2562 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2563 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2564 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2565 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2566 }
2567
2568 /* Commit it to the VMCS and update our cache. */
2569 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2570 AssertRCReturn(rc, rc);
2571 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2572
2573 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2574 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2575 return hmR0VmxSetupProcCtls2(pVCpu);
2576
2577 /* Sanity check, should not really happen. */
2578 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2579 {
2580 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2581 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2582 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2583 }
2584
2585 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2586 return VINF_SUCCESS;
2587}
2588
2589
2590/**
2591 * Sets up miscellaneous (everything other than Pin & Processor-based
2592 * VM-execution) control fields in the VMCS.
2593 *
2594 * @returns VBox status code.
2595 * @param pVCpu The cross context virtual CPU structure.
2596 */
2597static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2598{
2599 AssertPtr(pVCpu);
2600
2601 int rc = VERR_GENERAL_FAILURE;
2602
2603 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2604#if 0
2605 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2606 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2608
2609 /*
2610 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2611 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2612 * We thus use the exception bitmap to control it rather than use both.
2613 */
2614 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2615 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2616
2617 /* All IO & IOIO instructions cause VM-exits. */
2618 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2619 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2620
2621 /* Initialize the MSR-bitmap area. */
2622 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2623 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2624 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2625 AssertRCReturn(rc, rc);
2626#endif
2627
2628 /* Setup MSR auto-load/store area. */
2629 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2630 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2631 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2632 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2633 AssertRCReturn(rc, rc);
2634
2635 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2636 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2637 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2638 AssertRCReturn(rc, rc);
2639
2640 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2641 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2642 AssertRCReturn(rc, rc);
2643
2644 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2645#if 0
2646 /* Setup debug controls */
2647 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2648 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2649 AssertRCReturn(rc, rc);
2650#endif
2651
2652 return rc;
2653}
2654
2655
2656/**
2657 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2658 *
2659 * We shall setup those exception intercepts that don't change during the
2660 * lifetime of the VM here. The rest are done dynamically while loading the
2661 * guest state.
2662 *
2663 * @returns VBox status code.
2664 * @param pVCpu The cross context virtual CPU structure.
2665 */
2666static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2667{
2668 AssertPtr(pVCpu);
2669
2670 uint32_t uXcptBitmap;
2671
2672 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2673 uXcptBitmap = RT_BIT_32(X86_XCPT_AC);
2674
2675 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2676 and writes, and because recursive #DBs can cause the CPU hang, we must always
2677 intercept #DB. */
2678 uXcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2679
2680 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2681 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2682 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
2683
2684 /* Commit it to the VMCS. */
2685 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
2686 AssertRCReturn(rc, rc);
2687
2688 /* Update our cache of the exception bitmap. */
2689 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
2690 return VINF_SUCCESS;
2691}
2692
2693
2694/**
2695 * Does per-VM VT-x initialization.
2696 *
2697 * @returns VBox status code.
2698 * @param pVM The cross context VM structure.
2699 */
2700VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2701{
2702 LogFlowFunc(("pVM=%p\n", pVM));
2703
2704 int rc = hmR0VmxStructsAlloc(pVM);
2705 if (RT_FAILURE(rc))
2706 {
2707 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2708 return rc;
2709 }
2710
2711 return VINF_SUCCESS;
2712}
2713
2714
2715/**
2716 * Does per-VM VT-x termination.
2717 *
2718 * @returns VBox status code.
2719 * @param pVM The cross context VM structure.
2720 */
2721VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2722{
2723 LogFlowFunc(("pVM=%p\n", pVM));
2724
2725#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2726 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2727 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2728#endif
2729 hmR0VmxStructsFree(pVM);
2730 return VINF_SUCCESS;
2731}
2732
2733
2734/**
2735 * Sets up the VM for execution under VT-x.
2736 * This function is only called once per-VM during initialization.
2737 *
2738 * @returns VBox status code.
2739 * @param pVM The cross context VM structure.
2740 */
2741VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2742{
2743 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2744 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2745
2746 LogFlowFunc(("pVM=%p\n", pVM));
2747
2748 /*
2749 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2750 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2751 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2752 */
2753 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2754 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2755 || !pVM->hm.s.vmx.pRealModeTSS))
2756 {
2757 LogRelFunc(("Invalid real-on-v86 state.\n"));
2758 return VERR_INTERNAL_ERROR;
2759 }
2760
2761 /* Initialize these always, see hmR3InitFinalizeR0().*/
2762 pVM->hm.s.vmx.enmTlbFlushEpt = VMXTLBFLUSHEPT_NONE;
2763 pVM->hm.s.vmx.enmTlbFlushVpid = VMXTLBFLUSHVPID_NONE;
2764
2765 /* Setup the tagged-TLB flush handlers. */
2766 int rc = hmR0VmxSetupTaggedTlb(pVM);
2767 if (RT_FAILURE(rc))
2768 {
2769 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2770 return rc;
2771 }
2772
2773 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2774 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2775#if HC_ARCH_BITS == 64
2776 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2777 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2778 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2779 {
2780 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2781 }
2782#endif
2783
2784 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2785 RTCCUINTREG uHostCR4 = ASMGetCR4();
2786 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2787 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2788
2789 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2790 {
2791 PVMCPU pVCpu = &pVM->aCpus[i];
2792 AssertPtr(pVCpu);
2793 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2794
2795 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2796 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2797
2798 /* Set revision dword at the beginning of the VMCS structure. */
2799 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2800
2801 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2802 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2804 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2805
2806 /* Load this VMCS as the current VMCS. */
2807 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811 rc = hmR0VmxSetupPinCtls(pVCpu);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2814
2815 rc = hmR0VmxSetupProcCtls(pVCpu);
2816 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2817 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2818
2819 rc = hmR0VmxSetupMiscCtls(pVCpu);
2820 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2821 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2822
2823 rc = hmR0VmxInitXcptBitmap(pVCpu);
2824 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2825 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2826
2827#if HC_ARCH_BITS == 32
2828 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2829 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2830 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2831#endif
2832
2833 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2834 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2835 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2836 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2837
2838 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2839
2840 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2841 }
2842
2843 return VINF_SUCCESS;
2844}
2845
2846
2847/**
2848 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2849 * the VMCS.
2850 *
2851 * @returns VBox status code.
2852 */
2853static int hmR0VmxExportHostControlRegs(void)
2854{
2855 RTCCUINTREG uReg = ASMGetCR0();
2856 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2857 AssertRCReturn(rc, rc);
2858
2859 uReg = ASMGetCR3();
2860 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2861 AssertRCReturn(rc, rc);
2862
2863 uReg = ASMGetCR4();
2864 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2865 AssertRCReturn(rc, rc);
2866 return rc;
2867}
2868
2869
2870/**
2871 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2872 * the host-state area in the VMCS.
2873 *
2874 * @returns VBox status code.
2875 * @param pVCpu The cross context virtual CPU structure.
2876 */
2877static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2878{
2879#if HC_ARCH_BITS == 64
2880/**
2881 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2882 * requirements. See hmR0VmxExportHostSegmentRegs().
2883 */
2884# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2885 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2886 { \
2887 bool fValidSelector = true; \
2888 if ((selValue) & X86_SEL_LDT) \
2889 { \
2890 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2891 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2892 } \
2893 if (fValidSelector) \
2894 { \
2895 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2896 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2897 } \
2898 (selValue) = 0; \
2899 }
2900
2901 /*
2902 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2903 * should -not- save the messed up state without restoring the original host-state,
2904 * see @bugref{7240}.
2905 *
2906 * This apparently can happen (most likely the FPU changes), deal with it rather than
2907 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2908 */
2909 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2910 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2911 {
2912 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2913 pVCpu->idCpu));
2914 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2915 }
2916 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2917#else
2918 RT_NOREF(pVCpu);
2919#endif
2920
2921 /*
2922 * Host DS, ES, FS and GS segment registers.
2923 */
2924#if HC_ARCH_BITS == 64
2925 RTSEL uSelDS = ASMGetDS();
2926 RTSEL uSelES = ASMGetES();
2927 RTSEL uSelFS = ASMGetFS();
2928 RTSEL uSelGS = ASMGetGS();
2929#else
2930 RTSEL uSelDS = 0;
2931 RTSEL uSelES = 0;
2932 RTSEL uSelFS = 0;
2933 RTSEL uSelGS = 0;
2934#endif
2935
2936 /*
2937 * Host CS and SS segment registers.
2938 */
2939 RTSEL uSelCS = ASMGetCS();
2940 RTSEL uSelSS = ASMGetSS();
2941
2942 /*
2943 * Host TR segment register.
2944 */
2945 RTSEL uSelTR = ASMGetTR();
2946
2947#if HC_ARCH_BITS == 64
2948 /*
2949 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2950 * gain VM-entry and restore them before we get preempted.
2951 *
2952 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2953 */
2954 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2955 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2956 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2957 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2958# undef VMXLOCAL_ADJUST_HOST_SEG
2959#endif
2960
2961 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2962 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2963 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2964 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2965 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2966 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2967 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2968 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2969 Assert(uSelCS);
2970 Assert(uSelTR);
2971
2972 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2973#if 0
2974 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2975 Assert(uSelSS != 0);
2976#endif
2977
2978 /* Write these host selector fields into the host-state area in the VMCS. */
2979 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2980 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2981#if HC_ARCH_BITS == 64
2982 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2983 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2984 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2985 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2986#else
2987 NOREF(uSelDS);
2988 NOREF(uSelES);
2989 NOREF(uSelFS);
2990 NOREF(uSelGS);
2991#endif
2992 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2993 AssertRCReturn(rc, rc);
2994
2995 /*
2996 * Host GDTR and IDTR.
2997 */
2998 RTGDTR Gdtr;
2999 RTIDTR Idtr;
3000 RT_ZERO(Gdtr);
3001 RT_ZERO(Idtr);
3002 ASMGetGDTR(&Gdtr);
3003 ASMGetIDTR(&Idtr);
3004 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
3005 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
3006 AssertRCReturn(rc, rc);
3007
3008#if HC_ARCH_BITS == 64
3009 /*
3010 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
3011 * them to the maximum limit (0xffff) on every VM-exit.
3012 */
3013 if (Gdtr.cbGdt != 0xffff)
3014 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
3015
3016 /*
3017 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
3018 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
3019 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
3020 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
3021 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
3022 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3023 * at 0xffff on hosts where we are sure it won't cause trouble.
3024 */
3025# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3026 if (Idtr.cbIdt < 0x0fff)
3027# else
3028 if (Idtr.cbIdt != 0xffff)
3029# endif
3030 {
3031 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3032 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3033 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3034 }
3035#endif
3036
3037 /*
3038 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3039 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3040 * RPL should be too in most cases.
3041 */
3042 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3043 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3044
3045 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3046#if HC_ARCH_BITS == 64
3047 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3048
3049 /*
3050 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3051 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3052 * restoration if the host has something else. Task switching is not supported in 64-bit
3053 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3054 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3055 *
3056 * [1] See Intel spec. 3.5 "System Descriptor Types".
3057 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3058 */
3059 PVM pVM = pVCpu->CTX_SUFF(pVM);
3060 Assert(pDesc->System.u4Type == 11);
3061 if ( pDesc->System.u16LimitLow != 0x67
3062 || pDesc->System.u4LimitHigh)
3063 {
3064 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3065 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3066 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3067 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3068 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3069 }
3070
3071 /*
3072 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3073 */
3074 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3075 {
3076 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3077 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3078 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3079 {
3080 /* The GDT is read-only but the writable GDT is available. */
3081 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3082 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3083 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3084 AssertRCReturn(rc, rc);
3085 }
3086 }
3087#else
3088 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3089#endif
3090 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3091 AssertRCReturn(rc, rc);
3092
3093 /*
3094 * Host FS base and GS base.
3095 */
3096#if HC_ARCH_BITS == 64
3097 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3098 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3099 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3100 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3101 AssertRCReturn(rc, rc);
3102
3103 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3104 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3105 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3106 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3107 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3108#endif
3109 return VINF_SUCCESS;
3110}
3111
3112
3113/**
3114 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3115 * host-state area of the VMCS.
3116 *
3117 * Theses MSRs will be automatically restored on the host after every successful
3118 * VM-exit.
3119 *
3120 * @returns VBox status code.
3121 * @param pVCpu The cross context virtual CPU structure.
3122 *
3123 * @remarks No-long-jump zone!!!
3124 */
3125static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3126{
3127 AssertPtr(pVCpu);
3128 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3129
3130 /*
3131 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3132 * rather than swapping them on every VM-entry.
3133 */
3134 hmR0VmxLazySaveHostMsrs(pVCpu);
3135
3136 /*
3137 * Host Sysenter MSRs.
3138 */
3139 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3140#if HC_ARCH_BITS == 32
3141 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3142 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3143#else
3144 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3145 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3146#endif
3147 AssertRCReturn(rc, rc);
3148
3149 /*
3150 * Host EFER MSR.
3151 *
3152 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3153 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3154 */
3155 PVM pVM = pVCpu->CTX_SUFF(pVM);
3156 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3157 {
3158 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3159 AssertRCReturn(rc, rc);
3160 }
3161
3162 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3163
3164 return VINF_SUCCESS;
3165}
3166
3167
3168/**
3169 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3170 *
3171 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3172 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3173 * hmR0VMxExportGuestEntryCtls().
3174 *
3175 * @returns true if we need to load guest EFER, false otherwise.
3176 * @param pVCpu The cross context virtual CPU structure.
3177 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3178 * out-of-sync. Make sure to update the required fields
3179 * before using them.
3180 *
3181 * @remarks Requires EFER, CR4.
3182 * @remarks No-long-jump zone!!!
3183 */
3184static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3185{
3186#ifdef HMVMX_ALWAYS_SWAP_EFER
3187 return true;
3188#endif
3189
3190#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3191 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3192 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3193 return false;
3194#endif
3195
3196 PVM pVM = pVCpu->CTX_SUFF(pVM);
3197 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3198 uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3199
3200 /*
3201 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3202 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3203 */
3204 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3205 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3206 {
3207 return true;
3208 }
3209
3210 /*
3211 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3212 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3213 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3214 */
3215 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3216 && (pMixedCtx->cr0 & X86_CR0_PG)
3217 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3218 {
3219 /* Assert that host is PAE capable. */
3220 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3221 return true;
3222 }
3223
3224 return false;
3225}
3226
3227
3228/**
3229 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3230 *
3231 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3232 * see Intel spec. 24.8.1 "VM-entry controls".
3233 *
3234 * @returns VBox status code.
3235 * @param pVCpu The cross context virtual CPU structure.
3236 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3237 * out-of-sync. Make sure to update the required fields
3238 * before using them.
3239 *
3240 * @remarks Requires EFER.
3241 * @remarks No-long-jump zone!!!
3242 */
3243static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3244{
3245 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3246 {
3247 PVM pVM = pVCpu->CTX_SUFF(pVM);
3248 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3249 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3250
3251 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3252 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3253
3254 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3255 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3256 {
3257 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3258 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3259 }
3260 else
3261 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3262
3263 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3264 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3265 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3266 {
3267 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3268 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3269 }
3270
3271 /*
3272 * The following should -not- be set (since we're not in SMM mode):
3273 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3274 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3275 */
3276
3277 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3278 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3279
3280 if ((fVal & fZap) != fVal)
3281 {
3282 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3283 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3284 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3285 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3286 }
3287
3288 /* Commit it to the VMCS and update our cache. */
3289 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3290 {
3291 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3292 AssertRCReturn(rc, rc);
3293 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3294 }
3295
3296 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3297 }
3298 return VINF_SUCCESS;
3299}
3300
3301
3302/**
3303 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3304 *
3305 * @returns VBox status code.
3306 * @param pVCpu The cross context virtual CPU structure.
3307 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3308 * out-of-sync. Make sure to update the required fields
3309 * before using them.
3310 *
3311 * @remarks Requires EFER.
3312 */
3313static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3314{
3315 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3316 {
3317 PVM pVM = pVCpu->CTX_SUFF(pVM);
3318 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3319 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3320
3321 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3322 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3323
3324 /*
3325 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3326 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3327 * hmR0VmxExportHostMsrs().
3328 */
3329#if HC_ARCH_BITS == 64
3330 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3331 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3332#else
3333 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3334 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3335 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3336 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3337 {
3338 /* The switcher returns to long mode, EFER is managed by the switcher. */
3339 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3340 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3341 }
3342 else
3343 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3344#endif
3345
3346 /* If the newer VMCS fields for managing EFER exists, use it. */
3347 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3348 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3349 {
3350 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3351 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3352 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3353 }
3354
3355 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3356 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3357
3358 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3359 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3360 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3361
3362 /* Enable saving of the VMX preemption timer value on VM-exit. */
3363 if ( pVM->hm.s.vmx.fUsePreemptTimer
3364 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3365 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3366
3367 if ((fVal & fZap) != fVal)
3368 {
3369 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3370 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3371 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3372 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3373 }
3374
3375 /* Commit it to the VMCS and update our cache. */
3376 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3377 {
3378 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3379 AssertRCReturn(rc, rc);
3380 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3381 }
3382
3383 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3384 }
3385 return VINF_SUCCESS;
3386}
3387
3388
3389/**
3390 * Sets the TPR threshold in the VMCS.
3391 *
3392 * @returns VBox status code.
3393 * @param pVCpu The cross context virtual CPU structure.
3394 * @param u32TprThreshold The TPR threshold (task-priority class only).
3395 */
3396DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3397{
3398 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3399 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3400 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3401}
3402
3403
3404/**
3405 * Exports the guest APIC TPR state into the VMCS.
3406 *
3407 * @returns VBox status code.
3408 * @param pVCpu The cross context virtual CPU structure.
3409 *
3410 * @remarks No-long-jump zone!!!
3411 */
3412static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3413{
3414 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3415 {
3416 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3417
3418 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3419 && APICIsEnabled(pVCpu))
3420 {
3421 /*
3422 * Setup TPR shadowing.
3423 */
3424 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3425 {
3426 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3427
3428 bool fPendingIntr = false;
3429 uint8_t u8Tpr = 0;
3430 uint8_t u8PendingIntr = 0;
3431 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3432 AssertRCReturn(rc, rc);
3433
3434 /*
3435 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3436 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3437 * priority of the pending interrupt so we can deliver the interrupt. If there
3438 * are no interrupts pending, set threshold to 0 to not cause any
3439 * TPR-below-threshold VM-exits.
3440 */
3441 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3442 uint32_t u32TprThreshold = 0;
3443 if (fPendingIntr)
3444 {
3445 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3446 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3447 const uint8_t u8TprPriority = u8Tpr >> 4;
3448 if (u8PendingPriority <= u8TprPriority)
3449 u32TprThreshold = u8PendingPriority;
3450 }
3451
3452 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3453 AssertRCReturn(rc, rc);
3454 }
3455 }
3456 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3457 }
3458 return VINF_SUCCESS;
3459}
3460
3461
3462/**
3463 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3464 *
3465 * @returns Guest's interruptibility-state.
3466 * @param pVCpu The cross context virtual CPU structure.
3467 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3468 * out-of-sync. Make sure to update the required fields
3469 * before using them.
3470 *
3471 * @remarks No-long-jump zone!!!
3472 */
3473static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3474{
3475 /*
3476 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3477 */
3478 uint32_t fIntrState = 0;
3479 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3480 {
3481 /* If inhibition is active, RIP & RFLAGS should've been accessed
3482 (i.e. read previously from the VMCS or from ring-3). */
3483#ifdef VBOX_STRICT
3484 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3485 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3486#endif
3487 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3488 {
3489 if (pMixedCtx->eflags.Bits.u1IF)
3490 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3491 else
3492 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3493 }
3494 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3495 {
3496 /*
3497 * We can clear the inhibit force flag as even if we go back to the recompiler
3498 * without executing guest code in VT-x, the flag's condition to be cleared is
3499 * met and thus the cleared state is correct.
3500 */
3501 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3502 }
3503 }
3504
3505 /*
3506 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3507 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3508 * setting this would block host-NMIs and IRET will not clear the blocking.
3509 *
3510 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3511 */
3512 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3513 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3514 {
3515 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3516 }
3517
3518 return fIntrState;
3519}
3520
3521
3522/**
3523 * Exports the guest's interruptibility-state into the guest-state area in the
3524 * VMCS.
3525 *
3526 * @returns VBox status code.
3527 * @param pVCpu The cross context virtual CPU structure.
3528 * @param fIntrState The interruptibility-state to set.
3529 */
3530static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3531{
3532 NOREF(pVCpu);
3533 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3534 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3535 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3536}
3537
3538
3539/**
3540 * Exports the exception intercepts required for guest execution in the VMCS.
3541 *
3542 * @returns VBox status code.
3543 * @param pVCpu The cross context virtual CPU structure.
3544 *
3545 * @remarks No-long-jump zone!!!
3546 */
3547static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3548{
3549 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3550 {
3551 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3552
3553 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3554 if (pVCpu->hm.s.fGIMTrapXcptUD)
3555 uXcptBitmap |= RT_BIT(X86_XCPT_UD);
3556#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3557 else
3558 uXcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3559#endif
3560
3561 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_AC));
3562 Assert(uXcptBitmap & RT_BIT_32(X86_XCPT_DB));
3563
3564 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3565 {
3566 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3567 AssertRCReturn(rc, rc);
3568 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3569 }
3570
3571 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3572 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", uXcptBitmap));
3573 }
3574 return VINF_SUCCESS;
3575}
3576
3577
3578/**
3579 * Exports the guest's RIP into the guest-state area in the VMCS.
3580 *
3581 * @returns VBox status code.
3582 * @param pVCpu The cross context virtual CPU structure.
3583 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3584 * out-of-sync. Make sure to update the required fields
3585 * before using them.
3586 *
3587 * @remarks No-long-jump zone!!!
3588 */
3589static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3590{
3591 int rc = VINF_SUCCESS;
3592 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3593 {
3594 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3595
3596 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3597 AssertRCReturn(rc, rc);
3598
3599 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3600 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3601 }
3602 return rc;
3603}
3604
3605
3606/**
3607 * Exports the guest's RSP into the guest-state area in the VMCS.
3608 *
3609 * @returns VBox status code.
3610 * @param pVCpu The cross context virtual CPU structure.
3611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3612 * out-of-sync. Make sure to update the required fields
3613 * before using them.
3614 *
3615 * @remarks No-long-jump zone!!!
3616 */
3617static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3618{
3619 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3620 {
3621 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3622
3623 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3624 AssertRCReturn(rc, rc);
3625
3626 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3627 }
3628 return VINF_SUCCESS;
3629}
3630
3631
3632/**
3633 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3634 *
3635 * @returns VBox status code.
3636 * @param pVCpu The cross context virtual CPU structure.
3637 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3638 * out-of-sync. Make sure to update the required fields
3639 * before using them.
3640 *
3641 * @remarks No-long-jump zone!!!
3642 */
3643static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3644{
3645 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3646 {
3647 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3648
3649 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3650 Let us assert it as such and use 32-bit VMWRITE. */
3651 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3652 X86EFLAGS fEFlags = pMixedCtx->eflags;
3653 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3654 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3655
3656 /*
3657 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3658 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3659 * can run the real-mode guest code under Virtual 8086 mode.
3660 */
3661 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3662 {
3663 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3664 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3665 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3666 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3667 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3668 }
3669
3670 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3671 AssertRCReturn(rc, rc);
3672
3673 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3674 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3675 }
3676 return VINF_SUCCESS;
3677}
3678
3679
3680/**
3681 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3682 *
3683 * The guest FPU state is always pre-loaded hence we don't need to bother about
3684 * sharing FPU related CR0 bits between the guest and host.
3685 *
3686 * @returns VBox status code.
3687 * @param pVCpu The cross context virtual CPU structure.
3688 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3689 * out-of-sync. Make sure to update the required fields
3690 * before using them.
3691 *
3692 * @remarks No-long-jump zone!!!
3693 */
3694static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3695{
3696 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3697 {
3698 PVM pVM = pVCpu->CTX_SUFF(pVM);
3699 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3700 Assert(!RT_HI_U32(pMixedCtx->cr0));
3701
3702 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3703 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3704
3705 /*
3706 * Setup VT-x's view of the guest CR0.
3707 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3708 */
3709 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3710 if (pVM->hm.s.fNestedPaging)
3711 {
3712 if (CPUMIsGuestPagingEnabled(pVCpu))
3713 {
3714 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3715 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3716 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3717 }
3718 else
3719 {
3720 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3721 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3722 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3723 }
3724
3725 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3726 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3727 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3728 }
3729 else
3730 {
3731 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3732 u32GuestCr0 |= X86_CR0_WP;
3733 }
3734
3735 /*
3736 * Guest FPU bits.
3737 *
3738 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3739 * using CR0.TS.
3740 *
3741 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3742 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3743 */
3744 u32GuestCr0 |= X86_CR0_NE;
3745
3746 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3747 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3748
3749 /*
3750 * Update exception intercepts.
3751 */
3752 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3753 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3754 {
3755 Assert(PDMVmmDevHeapIsEnabled(pVM));
3756 Assert(pVM->hm.s.vmx.pRealModeTSS);
3757 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3758 }
3759 else
3760 {
3761 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3762 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3763 if (fInterceptMF)
3764 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3765 }
3766
3767 /* Additional intercepts for debugging, define these yourself explicitly. */
3768#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3769 uXcptBitmap |= 0
3770 | RT_BIT(X86_XCPT_BP)
3771 | RT_BIT(X86_XCPT_DE)
3772 | RT_BIT(X86_XCPT_NM)
3773 | RT_BIT(X86_XCPT_TS)
3774 | RT_BIT(X86_XCPT_UD)
3775 | RT_BIT(X86_XCPT_NP)
3776 | RT_BIT(X86_XCPT_SS)
3777 | RT_BIT(X86_XCPT_GP)
3778 | RT_BIT(X86_XCPT_PF)
3779 | RT_BIT(X86_XCPT_MF)
3780 ;
3781#elif defined(HMVMX_ALWAYS_TRAP_PF)
3782 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3783#endif
3784 Assert(pVM->hm.s.fNestedPaging || (uXcptBitmap & RT_BIT(X86_XCPT_PF)));
3785
3786 /*
3787 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3788 */
3789 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3790 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3791 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3792 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3793 else
3794 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3795
3796 u32GuestCr0 |= fSetCr0;
3797 u32GuestCr0 &= fZapCr0;
3798 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3799
3800 /*
3801 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3802 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3803 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3804 */
3805 uint32_t u32Cr0Mask = X86_CR0_PE
3806 | X86_CR0_NE
3807 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3808 | X86_CR0_PG
3809 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3810 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3811 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3812
3813 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3814 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3815 * and @bugref{6944}. */
3816#if 0
3817 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3818 u32Cr0Mask &= ~X86_CR0_PE;
3819#endif
3820 /*
3821 * Finally, update VMCS fields with the CR0 values and the exception bitmap.
3822 */
3823 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3824 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3825 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3826 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3827 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3828 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3829 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3830 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, uXcptBitmap);
3831 AssertRCReturn(rc, rc);
3832
3833 /* Update our caches. */
3834 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3835 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3836 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3837
3838 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3839
3840 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3841 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3842 }
3843
3844 return VINF_SUCCESS;
3845}
3846
3847
3848/**
3849 * Exports the guest control registers (CR3, CR4) into the guest-state area
3850 * in the VMCS.
3851 *
3852 * @returns VBox strict status code.
3853 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3854 * without unrestricted guest access and the VMMDev is not presently
3855 * mapped (e.g. EFI32).
3856 *
3857 * @param pVCpu The cross context virtual CPU structure.
3858 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3859 * out-of-sync. Make sure to update the required fields
3860 * before using them.
3861 *
3862 * @remarks No-long-jump zone!!!
3863 */
3864static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3865{
3866 int rc = VINF_SUCCESS;
3867 PVM pVM = pVCpu->CTX_SUFF(pVM);
3868
3869 /*
3870 * Guest CR2.
3871 * It's always loaded in the assembler code. Nothing to do here.
3872 */
3873
3874 /*
3875 * Guest CR3.
3876 */
3877 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3878 {
3879 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3880
3881 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3882 if (pVM->hm.s.fNestedPaging)
3883 {
3884 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3885
3886 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3887 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3888 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3889 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3890
3891 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3892 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3893 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3894
3895 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3896 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3897 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3898 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3899 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3900 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3901 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3902
3903 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3904 AssertRCReturn(rc, rc);
3905
3906 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3907 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3908 {
3909 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3910 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3911 {
3912 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3913 AssertRCReturn(rc, rc);
3914 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3915 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3916 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3917 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3918 AssertRCReturn(rc, rc);
3919 }
3920
3921 /*
3922 * The guest's view of its CR3 is unblemished with Nested Paging when the
3923 * guest is using paging or we have unrestricted guest execution to handle
3924 * the guest when it's not using paging.
3925 */
3926 GCPhysGuestCR3 = pMixedCtx->cr3;
3927 }
3928 else
3929 {
3930 /*
3931 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3932 * thinks it accesses physical memory directly, we use our identity-mapped
3933 * page table to map guest-linear to guest-physical addresses. EPT takes care
3934 * of translating it to host-physical addresses.
3935 */
3936 RTGCPHYS GCPhys;
3937 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3938
3939 /* We obtain it here every time as the guest could have relocated this PCI region. */
3940 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3941 if (RT_SUCCESS(rc))
3942 { /* likely */ }
3943 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3944 {
3945 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3946 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3947 }
3948 else
3949 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3950
3951 GCPhysGuestCR3 = GCPhys;
3952 }
3953
3954 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3955 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3956 AssertRCReturn(rc, rc);
3957 }
3958 else
3959 {
3960 /* Non-nested paging case, just use the hypervisor's CR3. */
3961 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3962
3963 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3964 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3965 AssertRCReturn(rc, rc);
3966 }
3967
3968 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3969 }
3970
3971 /*
3972 * Guest CR4.
3973 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3974 */
3975 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3976 {
3977 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3978 Assert(!RT_HI_U32(pMixedCtx->cr4));
3979
3980 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3981 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3982
3983 /*
3984 * Setup VT-x's view of the guest CR4.
3985 *
3986 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3987 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3988 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3989 *
3990 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3991 */
3992 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3993 {
3994 Assert(pVM->hm.s.vmx.pRealModeTSS);
3995 Assert(PDMVmmDevHeapIsEnabled(pVM));
3996 u32GuestCr4 &= ~X86_CR4_VME;
3997 }
3998
3999 if (pVM->hm.s.fNestedPaging)
4000 {
4001 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
4002 && !pVM->hm.s.vmx.fUnrestrictedGuest)
4003 {
4004 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
4005 u32GuestCr4 |= X86_CR4_PSE;
4006 /* Our identity mapping is a 32-bit page directory. */
4007 u32GuestCr4 &= ~X86_CR4_PAE;
4008 }
4009 /* else use guest CR4.*/
4010 }
4011 else
4012 {
4013 /*
4014 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
4015 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
4016 */
4017 switch (pVCpu->hm.s.enmShadowMode)
4018 {
4019 case PGMMODE_REAL: /* Real-mode. */
4020 case PGMMODE_PROTECTED: /* Protected mode without paging. */
4021 case PGMMODE_32_BIT: /* 32-bit paging. */
4022 {
4023 u32GuestCr4 &= ~X86_CR4_PAE;
4024 break;
4025 }
4026
4027 case PGMMODE_PAE: /* PAE paging. */
4028 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4029 {
4030 u32GuestCr4 |= X86_CR4_PAE;
4031 break;
4032 }
4033
4034 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4035 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4036#ifdef VBOX_ENABLE_64_BITS_GUESTS
4037 break;
4038#endif
4039 default:
4040 AssertFailed();
4041 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4042 }
4043 }
4044
4045 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4046 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4047 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4048 u32GuestCr4 |= fSetCr4;
4049 u32GuestCr4 &= fZapCr4;
4050
4051 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4052 that would cause a VM-exit. */
4053 uint32_t u32Cr4Mask = X86_CR4_VME
4054 | X86_CR4_PAE
4055 | X86_CR4_PGE
4056 | X86_CR4_PSE
4057 | X86_CR4_VMXE;
4058 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4059 u32Cr4Mask |= X86_CR4_OSXSAVE;
4060 if (pVM->cpum.ro.GuestFeatures.fPcid)
4061 u32Cr4Mask |= X86_CR4_PCIDE;
4062
4063 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4064 into the VMCS and update our cache. */
4065 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4066 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4067 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4068 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4069 AssertRCReturn(rc, rc);
4070 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4071
4072 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4073 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4074
4075 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4076
4077 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4078 fZapCr4));
4079 }
4080 return rc;
4081}
4082
4083
4084/**
4085 * Exports the guest debug registers into the guest-state area in the VMCS.
4086 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4087 *
4088 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4089 *
4090 * @returns VBox status code.
4091 * @param pVCpu The cross context virtual CPU structure.
4092 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4093 * out-of-sync. Make sure to update the required fields
4094 * before using them.
4095 *
4096 * @remarks No-long-jump zone!!!
4097 */
4098static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4099{
4100 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4101
4102#ifdef VBOX_STRICT
4103 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4104 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4105 {
4106 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4107 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4108 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4109 }
4110#endif
4111
4112 bool fSteppingDB = false;
4113 bool fInterceptMovDRx = false;
4114 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4115 if (pVCpu->hm.s.fSingleInstruction)
4116 {
4117 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4118 PVM pVM = pVCpu->CTX_SUFF(pVM);
4119 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4120 {
4121 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4122 Assert(fSteppingDB == false);
4123 }
4124 else
4125 {
4126 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4127 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4128 pVCpu->hm.s.fClearTrapFlag = true;
4129 fSteppingDB = true;
4130 }
4131 }
4132
4133 uint32_t u32GuestDr7;
4134 if ( fSteppingDB
4135 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4136 {
4137 /*
4138 * Use the combined guest and host DRx values found in the hypervisor register set
4139 * because the debugger has breakpoints active or someone is single stepping on the
4140 * host side without a monitor trap flag.
4141 *
4142 * Note! DBGF expects a clean DR6 state before executing guest code.
4143 */
4144#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4145 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4146 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4147 {
4148 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4149 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4150 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4151 }
4152 else
4153#endif
4154 if (!CPUMIsHyperDebugStateActive(pVCpu))
4155 {
4156 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4157 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4158 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4159 }
4160
4161 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4162 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4163 pVCpu->hm.s.fUsingHyperDR7 = true;
4164 fInterceptMovDRx = true;
4165 }
4166 else
4167 {
4168 /*
4169 * If the guest has enabled debug registers, we need to load them prior to
4170 * executing guest code so they'll trigger at the right time.
4171 */
4172 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4173 {
4174#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4175 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4176 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4177 {
4178 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4179 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4180 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4181 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4182 }
4183 else
4184#endif
4185 if (!CPUMIsGuestDebugStateActive(pVCpu))
4186 {
4187 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4188 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4189 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4190 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4191 }
4192 Assert(!fInterceptMovDRx);
4193 }
4194 /*
4195 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4196 * must intercept #DB in order to maintain a correct DR6 guest value, and
4197 * because we need to intercept it to prevent nested #DBs from hanging the
4198 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4199 */
4200#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4201 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4202 && !CPUMIsGuestDebugStateActive(pVCpu))
4203#else
4204 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4205#endif
4206 {
4207 fInterceptMovDRx = true;
4208 }
4209
4210 /* Update DR7 with the actual guest value. */
4211 u32GuestDr7 = pMixedCtx->dr[7];
4212 pVCpu->hm.s.fUsingHyperDR7 = false;
4213 }
4214
4215 if (fInterceptMovDRx)
4216 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4217 else
4218 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4219
4220 /*
4221 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4222 * monitor-trap flag and update our cache.
4223 */
4224 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4225 {
4226 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4227 AssertRCReturn(rc2, rc2);
4228 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4229 }
4230
4231 /*
4232 * Update guest DR7.
4233 */
4234 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4235 AssertRCReturn(rc, rc);
4236
4237 return VINF_SUCCESS;
4238}
4239
4240
4241#ifdef VBOX_STRICT
4242/**
4243 * Strict function to validate segment registers.
4244 *
4245 * @param pVCpu The cross context virtual CPU structure.
4246 * @param pCtx Pointer to the guest-CPU context.
4247 *
4248 * @remarks Will import guest CR0 on strict builds during validation of
4249 * segments.
4250 */
4251static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4252{
4253 /*
4254 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4255 *
4256 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4257 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4258 * and doesn't change the guest-context value.
4259 */
4260 PVM pVM = pVCpu->CTX_SUFF(pVM);
4261 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4262 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4263 && ( !CPUMIsGuestInRealModeEx(pCtx)
4264 && !CPUMIsGuestInV86ModeEx(pCtx)))
4265 {
4266 /* Protected mode checks */
4267 /* CS */
4268 Assert(pCtx->cs.Attr.n.u1Present);
4269 Assert(!(pCtx->cs.Attr.u & 0xf00));
4270 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4271 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4272 || !(pCtx->cs.Attr.n.u1Granularity));
4273 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4274 || (pCtx->cs.Attr.n.u1Granularity));
4275 /* CS cannot be loaded with NULL in protected mode. */
4276 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4277 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4278 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4279 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4280 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4281 else
4282 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4283 /* SS */
4284 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4285 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4286 if ( !(pCtx->cr0 & X86_CR0_PE)
4287 || pCtx->cs.Attr.n.u4Type == 3)
4288 {
4289 Assert(!pCtx->ss.Attr.n.u2Dpl);
4290 }
4291 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4292 {
4293 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4294 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4295 Assert(pCtx->ss.Attr.n.u1Present);
4296 Assert(!(pCtx->ss.Attr.u & 0xf00));
4297 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4298 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4299 || !(pCtx->ss.Attr.n.u1Granularity));
4300 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4301 || (pCtx->ss.Attr.n.u1Granularity));
4302 }
4303 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4304 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4305 {
4306 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4307 Assert(pCtx->ds.Attr.n.u1Present);
4308 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4309 Assert(!(pCtx->ds.Attr.u & 0xf00));
4310 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4311 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4312 || !(pCtx->ds.Attr.n.u1Granularity));
4313 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4314 || (pCtx->ds.Attr.n.u1Granularity));
4315 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4316 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4317 }
4318 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4319 {
4320 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4321 Assert(pCtx->es.Attr.n.u1Present);
4322 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4323 Assert(!(pCtx->es.Attr.u & 0xf00));
4324 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4325 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4326 || !(pCtx->es.Attr.n.u1Granularity));
4327 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4328 || (pCtx->es.Attr.n.u1Granularity));
4329 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4330 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4331 }
4332 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4333 {
4334 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4335 Assert(pCtx->fs.Attr.n.u1Present);
4336 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4337 Assert(!(pCtx->fs.Attr.u & 0xf00));
4338 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4339 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4340 || !(pCtx->fs.Attr.n.u1Granularity));
4341 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4342 || (pCtx->fs.Attr.n.u1Granularity));
4343 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4344 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4345 }
4346 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4347 {
4348 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4349 Assert(pCtx->gs.Attr.n.u1Present);
4350 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4351 Assert(!(pCtx->gs.Attr.u & 0xf00));
4352 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4353 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4354 || !(pCtx->gs.Attr.n.u1Granularity));
4355 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4356 || (pCtx->gs.Attr.n.u1Granularity));
4357 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4358 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4359 }
4360 /* 64-bit capable CPUs. */
4361# if HC_ARCH_BITS == 64
4362 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4363 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4364 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4365 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4366# endif
4367 }
4368 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4369 || ( CPUMIsGuestInRealModeEx(pCtx)
4370 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4371 {
4372 /* Real and v86 mode checks. */
4373 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4374 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4375 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4376 {
4377 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4378 }
4379 else
4380 {
4381 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4382 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4383 }
4384
4385 /* CS */
4386 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4387 Assert(pCtx->cs.u32Limit == 0xffff);
4388 Assert(u32CSAttr == 0xf3);
4389 /* SS */
4390 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4391 Assert(pCtx->ss.u32Limit == 0xffff);
4392 Assert(u32SSAttr == 0xf3);
4393 /* DS */
4394 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4395 Assert(pCtx->ds.u32Limit == 0xffff);
4396 Assert(u32DSAttr == 0xf3);
4397 /* ES */
4398 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4399 Assert(pCtx->es.u32Limit == 0xffff);
4400 Assert(u32ESAttr == 0xf3);
4401 /* FS */
4402 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4403 Assert(pCtx->fs.u32Limit == 0xffff);
4404 Assert(u32FSAttr == 0xf3);
4405 /* GS */
4406 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4407 Assert(pCtx->gs.u32Limit == 0xffff);
4408 Assert(u32GSAttr == 0xf3);
4409 /* 64-bit capable CPUs. */
4410# if HC_ARCH_BITS == 64
4411 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4412 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4413 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4414 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4415# endif
4416 }
4417}
4418#endif /* VBOX_STRICT */
4419
4420
4421/**
4422 * Exports a guest segment register into the guest-state area in the VMCS.
4423 *
4424 * @returns VBox status code.
4425 * @param pVCpu The cross context virtual CPU structure.
4426 * @param idxSel Index of the selector in the VMCS.
4427 * @param idxLimit Index of the segment limit in the VMCS.
4428 * @param idxBase Index of the segment base in the VMCS.
4429 * @param idxAccess Index of the access rights of the segment in the VMCS.
4430 * @param pSelReg Pointer to the segment selector.
4431 *
4432 * @remarks No-long-jump zone!!!
4433 */
4434static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4435 PCCPUMSELREG pSelReg)
4436{
4437 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4438 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4439 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4440 AssertRCReturn(rc, rc);
4441
4442 uint32_t u32Access = pSelReg->Attr.u;
4443 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4444 {
4445 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4446 u32Access = 0xf3;
4447 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4448 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4449 }
4450 else
4451 {
4452 /*
4453 * The way to differentiate between whether this is really a null selector or was just
4454 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4455 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4456 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4457 * NULL selectors loaded in protected-mode have their attribute as 0.
4458 */
4459 if (!u32Access)
4460 u32Access = X86DESCATTR_UNUSABLE;
4461 }
4462
4463 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4464 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4465 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4466
4467 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4468 AssertRCReturn(rc, rc);
4469 return rc;
4470}
4471
4472
4473/**
4474 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4475 * into the guest-state area in the VMCS.
4476 *
4477 * @returns VBox status code.
4478 * @param pVCpu The cross context virtual CPU structure.
4479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4480 * out-of-sync. Make sure to update the required fields
4481 * before using them.
4482 *
4483 * @remarks Will import guest CR0 on strict builds during validation of
4484 * segments.
4485 * @remarks No-long-jump zone!!!
4486 */
4487static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4488{
4489 int rc = VERR_INTERNAL_ERROR_5;
4490 PVM pVM = pVCpu->CTX_SUFF(pVM);
4491
4492 /*
4493 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4494 */
4495 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4496 {
4497#ifdef VBOX_WITH_REM
4498 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4499 {
4500 Assert(pVM->hm.s.vmx.pRealModeTSS);
4501 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4502 if ( pVCpu->hm.s.vmx.fWasInRealMode
4503 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4504 {
4505 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4506 in real-mode (e.g. OpenBSD 4.0) */
4507 REMFlushTBs(pVM);
4508 Log4Func(("Switch to protected mode detected!\n"));
4509 pVCpu->hm.s.vmx.fWasInRealMode = false;
4510 }
4511 }
4512#endif
4513 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4514 {
4515 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4516 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4517 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4518 rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4519 AssertRCReturn(rc, rc);
4520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4521 }
4522
4523 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4524 {
4525 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4526 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4527 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4528 rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4529 AssertRCReturn(rc, rc);
4530 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4531 }
4532
4533 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4534 {
4535 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4536 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4537 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4538 rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4539 AssertRCReturn(rc, rc);
4540 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4541 }
4542
4543 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4544 {
4545 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4546 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4547 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4548 rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4549 AssertRCReturn(rc, rc);
4550 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4551 }
4552
4553 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4554 {
4555 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4556 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4557 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4558 rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4559 AssertRCReturn(rc, rc);
4560 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4561 }
4562
4563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4564 {
4565 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4566 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4567 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4568 rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4569 AssertRCReturn(rc, rc);
4570 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4571 }
4572
4573#ifdef VBOX_STRICT
4574 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4575#endif
4576
4577 /* Update the exit history entry with the correct CS.BASE + RIP. */
4578 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4579 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4580
4581 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4582 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4583 }
4584
4585 /*
4586 * Guest TR.
4587 */
4588 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4589 {
4590 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4591
4592 /*
4593 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4594 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4595 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4596 */
4597 uint16_t u16Sel = 0;
4598 uint32_t u32Limit = 0;
4599 uint64_t u64Base = 0;
4600 uint32_t u32AccessRights = 0;
4601
4602 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4603 {
4604 u16Sel = pMixedCtx->tr.Sel;
4605 u32Limit = pMixedCtx->tr.u32Limit;
4606 u64Base = pMixedCtx->tr.u64Base;
4607 u32AccessRights = pMixedCtx->tr.Attr.u;
4608 }
4609 else
4610 {
4611 Assert(pVM->hm.s.vmx.pRealModeTSS);
4612 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4613
4614 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4615 RTGCPHYS GCPhys;
4616 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4617 AssertRCReturn(rc, rc);
4618
4619 X86DESCATTR DescAttr;
4620 DescAttr.u = 0;
4621 DescAttr.n.u1Present = 1;
4622 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4623
4624 u16Sel = 0;
4625 u32Limit = HM_VTX_TSS_SIZE;
4626 u64Base = GCPhys; /* in real-mode phys = virt. */
4627 u32AccessRights = DescAttr.u;
4628 }
4629
4630 /* Validate. */
4631 Assert(!(u16Sel & RT_BIT(2)));
4632 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4633 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4634 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4635 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4636 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4637 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4638 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4639 Assert( (u32Limit & 0xfff) == 0xfff
4640 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4641 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4642 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4643
4644 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4645 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4646 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4647 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4648 AssertRCReturn(rc, rc);
4649
4650 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4651 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4652 }
4653
4654 /*
4655 * Guest GDTR.
4656 */
4657 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4658 {
4659 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4660
4661 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4662 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4663 AssertRCReturn(rc, rc);
4664
4665 /* Validate. */
4666 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4667
4668 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4669 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4670 }
4671
4672 /*
4673 * Guest LDTR.
4674 */
4675 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4676 {
4677 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4678
4679 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4680 uint32_t u32Access = 0;
4681 if (!pMixedCtx->ldtr.Attr.u)
4682 u32Access = X86DESCATTR_UNUSABLE;
4683 else
4684 u32Access = pMixedCtx->ldtr.Attr.u;
4685
4686 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4687 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4688 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4689 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4690 AssertRCReturn(rc, rc);
4691
4692 /* Validate. */
4693 if (!(u32Access & X86DESCATTR_UNUSABLE))
4694 {
4695 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4696 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4697 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4698 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4699 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4700 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4701 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4702 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4703 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4704 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4705 }
4706
4707 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4708 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4709 }
4710
4711 /*
4712 * Guest IDTR.
4713 */
4714 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4715 {
4716 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4717
4718 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4719 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4720 AssertRCReturn(rc, rc);
4721
4722 /* Validate. */
4723 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4724
4725 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4726 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4727 }
4728
4729 return VINF_SUCCESS;
4730}
4731
4732
4733/**
4734 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4735 * areas.
4736 *
4737 * These MSRs will automatically be loaded to the host CPU on every successful
4738 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4739 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4740 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4741 *
4742 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4743 *
4744 * @returns VBox status code.
4745 * @param pVCpu The cross context virtual CPU structure.
4746 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4747 * out-of-sync. Make sure to update the required fields
4748 * before using them.
4749 *
4750 * @remarks No-long-jump zone!!!
4751 */
4752static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4753{
4754 AssertPtr(pVCpu);
4755 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4756
4757 /*
4758 * MSRs that we use the auto-load/store MSR area in the VMCS.
4759 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4760 */
4761 PVM pVM = pVCpu->CTX_SUFF(pVM);
4762 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4763 {
4764 if (pVM->hm.s.fAllow64BitGuests)
4765 {
4766#if HC_ARCH_BITS == 32
4767 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4768
4769 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4770 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4771 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4772 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4773 AssertRCReturn(rc, rc);
4774# ifdef LOG_ENABLED
4775 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4776 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4777 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4778# endif
4779#endif
4780 }
4781 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4782 }
4783
4784 /*
4785 * Guest Sysenter MSRs.
4786 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4787 * VM-exits on WRMSRs for these MSRs.
4788 */
4789 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4790 {
4791 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4792
4793 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4794 {
4795 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4796 AssertRCReturn(rc, rc);
4797 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4798 }
4799
4800 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4801 {
4802 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4803 AssertRCReturn(rc, rc);
4804 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4805 }
4806
4807 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4808 {
4809 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4810 AssertRCReturn(rc, rc);
4811 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4812 }
4813 }
4814
4815 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4816 {
4817 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4818
4819 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4820 {
4821 /*
4822 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4823 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4824 */
4825 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4826 {
4827 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4828 AssertRCReturn(rc,rc);
4829 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4830 }
4831 else
4832 {
4833 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4834 NULL /* pfAddedAndUpdated */);
4835 AssertRCReturn(rc, rc);
4836
4837 /* We need to intercept reads too, see @bugref{7386#c16}. */
4838 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4839 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4840 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4841 pVCpu->hm.s.vmx.cMsrs));
4842 }
4843 }
4844 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4845 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4846 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4847 }
4848
4849 return VINF_SUCCESS;
4850}
4851
4852
4853#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4854/**
4855 * Check if guest state allows safe use of 32-bit switcher again.
4856 *
4857 * Segment bases and protected mode structures must be 32-bit addressable
4858 * because the 32-bit switcher will ignore high dword when writing these VMCS
4859 * fields. See @bugref{8432} for details.
4860 *
4861 * @returns true if safe, false if must continue to use the 64-bit switcher.
4862 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4863 * out-of-sync. Make sure to update the required fields
4864 * before using them.
4865 *
4866 * @remarks No-long-jump zone!!!
4867 */
4868static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4869{
4870 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4871 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4872 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4873 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4874 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4875 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4876 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4877 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4878 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4879 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4880
4881 /* All good, bases are 32-bit. */
4882 return true;
4883}
4884#endif
4885
4886
4887/**
4888 * Selects up the appropriate function to run guest code.
4889 *
4890 * @returns VBox status code.
4891 * @param pVCpu The cross context virtual CPU structure.
4892 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4893 * out-of-sync. Make sure to update the required fields
4894 * before using them.
4895 *
4896 * @remarks No-long-jump zone!!!
4897 */
4898static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4899{
4900 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4901 {
4902#ifndef VBOX_ENABLE_64_BITS_GUESTS
4903 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4904#endif
4905 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4906#if HC_ARCH_BITS == 32
4907 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4908 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4909 {
4910#ifdef VBOX_STRICT
4911 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4912 {
4913 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4914 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4915 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4916 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4917 | HM_CHANGED_VMX_ENTRY_CTLS
4918 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4919 }
4920#endif
4921 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4922
4923 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4924 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4925 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4926 Log4Func(("Selected 64-bit switcher\n"));
4927 }
4928#else
4929 /* 64-bit host. */
4930 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4931#endif
4932 }
4933 else
4934 {
4935 /* Guest is not in long mode, use the 32-bit handler. */
4936#if HC_ARCH_BITS == 32
4937 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4938 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4939 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4940 {
4941# ifdef VBOX_STRICT
4942 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4943 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4944 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4945 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4946 | HM_CHANGED_VMX_ENTRY_CTLS
4947 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4948# endif
4949 }
4950# ifdef VBOX_ENABLE_64_BITS_GUESTS
4951 /*
4952 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4953 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4954 * switcher flag because now we know the guest is in a sane state where it's safe
4955 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4956 * the much faster 32-bit switcher again.
4957 */
4958 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4959 {
4960 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4961 Log4Func(("Selected 32-bit switcher\n"));
4962 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4963 }
4964 else
4965 {
4966 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4967 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4968 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4969 {
4970 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4971 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4972 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4973 | HM_CHANGED_VMX_ENTRY_CTLS
4974 | HM_CHANGED_VMX_EXIT_CTLS
4975 | HM_CHANGED_HOST_CONTEXT);
4976 Log4Func(("Selected 32-bit switcher (safe)\n"));
4977 }
4978 }
4979# else
4980 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4981# endif
4982#else
4983 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4984#endif
4985 }
4986 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4987 return VINF_SUCCESS;
4988}
4989
4990
4991/**
4992 * Wrapper for running the guest code in VT-x.
4993 *
4994 * @returns VBox status code, no informational status codes.
4995 * @param pVCpu The cross context virtual CPU structure.
4996 * @param pCtx Pointer to the guest-CPU context.
4997 *
4998 * @remarks No-long-jump zone!!!
4999 */
5000DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
5001{
5002 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
5003 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
5004
5005 /*
5006 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
5007 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
5008 * callee-saved and thus the need for this XMM wrapper.
5009 *
5010 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
5011 */
5012 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
5013 /** @todo Add stats for resume vs launch. */
5014 PVM pVM = pVCpu->CTX_SUFF(pVM);
5015#ifdef VBOX_WITH_KERNEL_USING_XMM
5016 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
5017#else
5018 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
5019#endif
5020 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
5021 return rc;
5022}
5023
5024
5025/**
5026 * Reports world-switch error and dumps some useful debug info.
5027 *
5028 * @param pVCpu The cross context virtual CPU structure.
5029 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5030 * @param pCtx Pointer to the guest-CPU context.
5031 * @param pVmxTransient Pointer to the VMX transient structure (only
5032 * exitReason updated).
5033 */
5034static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5035{
5036 Assert(pVCpu);
5037 Assert(pCtx);
5038 Assert(pVmxTransient);
5039 HMVMX_ASSERT_PREEMPT_SAFE();
5040
5041 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5042 switch (rcVMRun)
5043 {
5044 case VERR_VMX_INVALID_VMXON_PTR:
5045 AssertFailed();
5046 break;
5047 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5048 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5049 {
5050 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5051 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5052 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5053 AssertRC(rc);
5054
5055 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5056 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5057 Cannot do it here as we may have been long preempted. */
5058
5059#ifdef VBOX_STRICT
5060 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5061 pVmxTransient->uExitReason));
5062 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5063 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5064 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5065 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5066 else
5067 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5068 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5069 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5070
5071 /* VMX control bits. */
5072 uint32_t u32Val;
5073 uint64_t u64Val;
5074 RTHCUINTREG uHCReg;
5075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5076 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5078 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5079 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5080 {
5081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5083 }
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5088 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5089 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5090 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5091 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5092 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5093 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5094 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5095 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5096 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5097 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5098 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5099 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5100 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5101 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5102 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5103 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5104 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5105 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5106 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5107 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5108 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5109 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5110 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5111 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5112 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5113 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5114 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5115 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5116 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5117 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5118 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5119 {
5120 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5121 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5122 }
5123
5124 /* Guest bits. */
5125 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5126 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5127 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5128 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5129 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5130 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5131 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5132 {
5133 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5134 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5135 }
5136
5137 /* Host bits. */
5138 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5139 Log4(("Host CR0 %#RHr\n", uHCReg));
5140 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5141 Log4(("Host CR3 %#RHr\n", uHCReg));
5142 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5143 Log4(("Host CR4 %#RHr\n", uHCReg));
5144
5145 RTGDTR HostGdtr;
5146 PCX86DESCHC pDesc;
5147 ASMGetGDTR(&HostGdtr);
5148 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5149 Log4(("Host CS %#08x\n", u32Val));
5150 if (u32Val < HostGdtr.cbGdt)
5151 {
5152 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5153 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5154 }
5155
5156 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5157 Log4(("Host DS %#08x\n", u32Val));
5158 if (u32Val < HostGdtr.cbGdt)
5159 {
5160 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5161 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5162 }
5163
5164 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5165 Log4(("Host ES %#08x\n", u32Val));
5166 if (u32Val < HostGdtr.cbGdt)
5167 {
5168 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5169 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5170 }
5171
5172 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5173 Log4(("Host FS %#08x\n", u32Val));
5174 if (u32Val < HostGdtr.cbGdt)
5175 {
5176 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5177 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5178 }
5179
5180 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5181 Log4(("Host GS %#08x\n", u32Val));
5182 if (u32Val < HostGdtr.cbGdt)
5183 {
5184 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5185 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5186 }
5187
5188 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5189 Log4(("Host SS %#08x\n", u32Val));
5190 if (u32Val < HostGdtr.cbGdt)
5191 {
5192 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5193 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5194 }
5195
5196 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5197 Log4(("Host TR %#08x\n", u32Val));
5198 if (u32Val < HostGdtr.cbGdt)
5199 {
5200 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5201 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5202 }
5203
5204 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5205 Log4(("Host TR Base %#RHv\n", uHCReg));
5206 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5207 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5208 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5209 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5210 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5211 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5212 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5213 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5214 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5215 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5216 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5217 Log4(("Host RSP %#RHv\n", uHCReg));
5218 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5219 Log4(("Host RIP %#RHv\n", uHCReg));
5220# if HC_ARCH_BITS == 64
5221 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5222 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5223 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5224 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5225 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5226 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5227# endif
5228#endif /* VBOX_STRICT */
5229 break;
5230 }
5231
5232 default:
5233 /* Impossible */
5234 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5235 break;
5236 }
5237 NOREF(pCtx);
5238}
5239
5240
5241#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5242#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5243# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5244#endif
5245#ifdef VBOX_STRICT
5246static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5247{
5248 switch (idxField)
5249 {
5250 case VMX_VMCS_GUEST_RIP:
5251 case VMX_VMCS_GUEST_RSP:
5252 case VMX_VMCS_GUEST_SYSENTER_EIP:
5253 case VMX_VMCS_GUEST_SYSENTER_ESP:
5254 case VMX_VMCS_GUEST_GDTR_BASE:
5255 case VMX_VMCS_GUEST_IDTR_BASE:
5256 case VMX_VMCS_GUEST_CS_BASE:
5257 case VMX_VMCS_GUEST_DS_BASE:
5258 case VMX_VMCS_GUEST_ES_BASE:
5259 case VMX_VMCS_GUEST_FS_BASE:
5260 case VMX_VMCS_GUEST_GS_BASE:
5261 case VMX_VMCS_GUEST_SS_BASE:
5262 case VMX_VMCS_GUEST_LDTR_BASE:
5263 case VMX_VMCS_GUEST_TR_BASE:
5264 case VMX_VMCS_GUEST_CR3:
5265 return true;
5266 }
5267 return false;
5268}
5269
5270static bool hmR0VmxIsValidReadField(uint32_t idxField)
5271{
5272 switch (idxField)
5273 {
5274 /* Read-only fields. */
5275 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5276 return true;
5277 }
5278 /* Remaining readable fields should also be writable. */
5279 return hmR0VmxIsValidWriteField(idxField);
5280}
5281#endif /* VBOX_STRICT */
5282
5283
5284/**
5285 * Executes the specified handler in 64-bit mode.
5286 *
5287 * @returns VBox status code (no informational status codes).
5288 * @param pVCpu The cross context virtual CPU structure.
5289 * @param enmOp The operation to perform.
5290 * @param cParams Number of parameters.
5291 * @param paParam Array of 32-bit parameters.
5292 */
5293VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5294{
5295 PVM pVM = pVCpu->CTX_SUFF(pVM);
5296 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5297 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5298 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5299 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5300
5301#ifdef VBOX_STRICT
5302 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5303 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5304
5305 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5306 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5307#endif
5308
5309 /* Disable interrupts. */
5310 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5311
5312#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5313 RTCPUID idHostCpu = RTMpCpuId();
5314 CPUMR0SetLApic(pVCpu, idHostCpu);
5315#endif
5316
5317 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5318 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5319
5320 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5321 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5322 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5323
5324 /* Leave VMX Root Mode. */
5325 VMXDisable();
5326
5327 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5328
5329 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5330 CPUMSetHyperEIP(pVCpu, enmOp);
5331 for (int i = (int)cParams - 1; i >= 0; i--)
5332 CPUMPushHyper(pVCpu, paParam[i]);
5333
5334 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5335
5336 /* Call the switcher. */
5337 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5338 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5339
5340 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5341 /* Make sure the VMX instructions don't cause #UD faults. */
5342 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5343
5344 /* Re-enter VMX Root Mode */
5345 int rc2 = VMXEnable(HCPhysCpuPage);
5346 if (RT_FAILURE(rc2))
5347 {
5348 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5349 ASMSetFlags(fOldEFlags);
5350 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5351 return rc2;
5352 }
5353
5354 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5355 AssertRC(rc2);
5356 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5357 Assert(!(ASMGetFlags() & X86_EFL_IF));
5358 ASMSetFlags(fOldEFlags);
5359 return rc;
5360}
5361
5362
5363/**
5364 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5365 * supporting 64-bit guests.
5366 *
5367 * @returns VBox status code.
5368 * @param fResume Whether to VMLAUNCH or VMRESUME.
5369 * @param pCtx Pointer to the guest-CPU context.
5370 * @param pCache Pointer to the VMCS cache.
5371 * @param pVM The cross context VM structure.
5372 * @param pVCpu The cross context virtual CPU structure.
5373 */
5374DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5375{
5376 NOREF(fResume);
5377
5378 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5379 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5380
5381#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5382 pCache->uPos = 1;
5383 pCache->interPD = PGMGetInterPaeCR3(pVM);
5384 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5385#endif
5386
5387#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5388 pCache->TestIn.HCPhysCpuPage = 0;
5389 pCache->TestIn.HCPhysVmcs = 0;
5390 pCache->TestIn.pCache = 0;
5391 pCache->TestOut.HCPhysVmcs = 0;
5392 pCache->TestOut.pCache = 0;
5393 pCache->TestOut.pCtx = 0;
5394 pCache->TestOut.eflags = 0;
5395#else
5396 NOREF(pCache);
5397#endif
5398
5399 uint32_t aParam[10];
5400 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5401 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5402 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5403 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5404 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5405 aParam[5] = 0;
5406 aParam[6] = VM_RC_ADDR(pVM, pVM);
5407 aParam[7] = 0;
5408 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5409 aParam[9] = 0;
5410
5411#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5412 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5413 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5414#endif
5415 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5416
5417#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5418 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5419 Assert(pCtx->dr[4] == 10);
5420 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5421#endif
5422
5423#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5424 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5425 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5426 pVCpu->hm.s.vmx.HCPhysVmcs));
5427 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5428 pCache->TestOut.HCPhysVmcs));
5429 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5430 pCache->TestOut.pCache));
5431 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5432 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5433 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5434 pCache->TestOut.pCtx));
5435 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5436#endif
5437 NOREF(pCtx);
5438 return rc;
5439}
5440
5441
5442/**
5443 * Initialize the VMCS-Read cache.
5444 *
5445 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5446 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5447 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5448 * (those that have a 32-bit FULL & HIGH part).
5449 *
5450 * @returns VBox status code.
5451 * @param pVCpu The cross context virtual CPU structure.
5452 */
5453static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5454{
5455#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5456 do { \
5457 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5458 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5459 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5460 ++cReadFields; \
5461 } while (0)
5462
5463 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5464 uint32_t cReadFields = 0;
5465
5466 /*
5467 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5468 * and serve to indicate exceptions to the rules.
5469 */
5470
5471 /* Guest-natural selector base fields. */
5472#if 0
5473 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5474 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5475 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5476#endif
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5487 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5488 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5489#if 0
5490 /* Unused natural width guest-state fields. */
5491 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5493#endif
5494 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5495 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5496
5497 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5498 these 64-bit fields (using "FULL" and "HIGH" fields). */
5499#if 0
5500 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5501 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5502 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5503 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5504 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5505 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5506 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5507 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5508 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5509#endif
5510
5511 /* Natural width guest-state fields. */
5512 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5513#if 0
5514 /* Currently unused field. */
5515 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5516#endif
5517
5518 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5519 {
5520 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5521 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5522 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5523 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5524 }
5525 else
5526 {
5527 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5528 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5529 }
5530
5531#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5532 return VINF_SUCCESS;
5533}
5534
5535
5536/**
5537 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5538 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5539 * darwin, running 64-bit guests).
5540 *
5541 * @returns VBox status code.
5542 * @param pVCpu The cross context virtual CPU structure.
5543 * @param idxField The VMCS field encoding.
5544 * @param u64Val 16, 32 or 64-bit value.
5545 */
5546VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5547{
5548 int rc;
5549 switch (idxField)
5550 {
5551 /*
5552 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5553 */
5554 /* 64-bit Control fields. */
5555 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5556 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5557 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5558 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5559 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5560 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5561 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5562 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5563 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5564 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5565 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5566 case VMX_VMCS64_CTRL_EPTP_FULL:
5567 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5568 /* 64-bit Guest-state fields. */
5569 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5570 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5571 case VMX_VMCS64_GUEST_PAT_FULL:
5572 case VMX_VMCS64_GUEST_EFER_FULL:
5573 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5574 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5575 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5576 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5577 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5578 /* 64-bit Host-state fields. */
5579 case VMX_VMCS64_HOST_PAT_FULL:
5580 case VMX_VMCS64_HOST_EFER_FULL:
5581 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5582 {
5583 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5584 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5585 break;
5586 }
5587
5588 /*
5589 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5590 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5591 */
5592 /* Natural-width Guest-state fields. */
5593 case VMX_VMCS_GUEST_CR3:
5594 case VMX_VMCS_GUEST_ES_BASE:
5595 case VMX_VMCS_GUEST_CS_BASE:
5596 case VMX_VMCS_GUEST_SS_BASE:
5597 case VMX_VMCS_GUEST_DS_BASE:
5598 case VMX_VMCS_GUEST_FS_BASE:
5599 case VMX_VMCS_GUEST_GS_BASE:
5600 case VMX_VMCS_GUEST_LDTR_BASE:
5601 case VMX_VMCS_GUEST_TR_BASE:
5602 case VMX_VMCS_GUEST_GDTR_BASE:
5603 case VMX_VMCS_GUEST_IDTR_BASE:
5604 case VMX_VMCS_GUEST_RSP:
5605 case VMX_VMCS_GUEST_RIP:
5606 case VMX_VMCS_GUEST_SYSENTER_ESP:
5607 case VMX_VMCS_GUEST_SYSENTER_EIP:
5608 {
5609 if (!(RT_HI_U32(u64Val)))
5610 {
5611 /* If this field is 64-bit, VT-x will zero out the top bits. */
5612 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5613 }
5614 else
5615 {
5616 /* Assert that only the 32->64 switcher case should ever come here. */
5617 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5618 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5619 }
5620 break;
5621 }
5622
5623 default:
5624 {
5625 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5626 rc = VERR_INVALID_PARAMETER;
5627 break;
5628 }
5629 }
5630 AssertRCReturn(rc, rc);
5631 return rc;
5632}
5633
5634
5635/**
5636 * Queue up a VMWRITE by using the VMCS write cache.
5637 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5638 *
5639 * @param pVCpu The cross context virtual CPU structure.
5640 * @param idxField The VMCS field encoding.
5641 * @param u64Val 16, 32 or 64-bit value.
5642 */
5643VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5644{
5645 AssertPtr(pVCpu);
5646 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5647
5648 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5649 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5650
5651 /* Make sure there are no duplicates. */
5652 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5653 {
5654 if (pCache->Write.aField[i] == idxField)
5655 {
5656 pCache->Write.aFieldVal[i] = u64Val;
5657 return VINF_SUCCESS;
5658 }
5659 }
5660
5661 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5662 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5663 pCache->Write.cValidEntries++;
5664 return VINF_SUCCESS;
5665}
5666#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5667
5668
5669/**
5670 * Sets up the usage of TSC-offsetting and updates the VMCS.
5671 *
5672 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5673 * VMX preemption timer.
5674 *
5675 * @returns VBox status code.
5676 * @param pVCpu The cross context virtual CPU structure.
5677 *
5678 * @remarks No-long-jump zone!!!
5679 */
5680static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5681{
5682 bool fOffsettedTsc;
5683 bool fParavirtTsc;
5684 PVM pVM = pVCpu->CTX_SUFF(pVM);
5685 uint64_t uTscOffset;
5686 if (pVM->hm.s.vmx.fUsePreemptTimer)
5687 {
5688 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5689
5690 /* Make sure the returned values have sane upper and lower boundaries. */
5691 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5692 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5693 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5694 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5695
5696 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5697 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5698 AssertRC(rc);
5699 }
5700 else
5701 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5702
5703 /** @todo later optimize this to be done elsewhere and not before every
5704 * VM-entry. */
5705 if (fParavirtTsc)
5706 {
5707 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5708 information before every VM-entry, hence disable it for performance sake. */
5709#if 0
5710 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5711 AssertRC(rc);
5712#endif
5713 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5714 }
5715
5716 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5717 if ( fOffsettedTsc
5718 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5719 {
5720 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5721 {
5722 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5723 AssertRC(rc);
5724 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5725 }
5726
5727 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5728 {
5729 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5730 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5731 AssertRC(rc);
5732 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5733 }
5734 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5735 }
5736 else
5737 {
5738 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5739 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5740 {
5741 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5742 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5743 AssertRC(rc);
5744 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5745 }
5746 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5747 }
5748}
5749
5750
5751/**
5752 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5753 * VM-exit interruption info type.
5754 *
5755 * @returns The IEM exception flags.
5756 * @param uVector The event vector.
5757 * @param uVmxVectorType The VMX event type.
5758 *
5759 * @remarks This function currently only constructs flags required for
5760 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5761 * and CR2 aspects of an exception are not included).
5762 */
5763static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5764{
5765 uint32_t fIemXcptFlags;
5766 switch (uVmxVectorType)
5767 {
5768 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5769 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5770 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5771 break;
5772
5773 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5774 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5775 break;
5776
5777 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5778 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5779 break;
5780
5781 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5782 {
5783 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5784 if (uVector == X86_XCPT_BP)
5785 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5786 else if (uVector == X86_XCPT_OF)
5787 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5788 else
5789 {
5790 fIemXcptFlags = 0;
5791 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5792 }
5793 break;
5794 }
5795
5796 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5797 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5798 break;
5799
5800 default:
5801 fIemXcptFlags = 0;
5802 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5803 break;
5804 }
5805 return fIemXcptFlags;
5806}
5807
5808
5809/**
5810 * Sets an event as a pending event to be injected into the guest.
5811 *
5812 * @param pVCpu The cross context virtual CPU structure.
5813 * @param u32IntInfo The VM-entry interruption-information field.
5814 * @param cbInstr The VM-entry instruction length in bytes (for software
5815 * interrupts, exceptions and privileged software
5816 * exceptions).
5817 * @param u32ErrCode The VM-entry exception error code.
5818 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5819 * page-fault.
5820 *
5821 * @remarks Statistics counter assumes this is a guest event being injected or
5822 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5823 * always incremented.
5824 */
5825DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5826 RTGCUINTPTR GCPtrFaultAddress)
5827{
5828 Assert(!pVCpu->hm.s.Event.fPending);
5829 pVCpu->hm.s.Event.fPending = true;
5830 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5831 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5832 pVCpu->hm.s.Event.cbInstr = cbInstr;
5833 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5834}
5835
5836
5837/**
5838 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5839 *
5840 * @param pVCpu The cross context virtual CPU structure.
5841 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5842 * out-of-sync. Make sure to update the required fields
5843 * before using them.
5844 */
5845DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5846{
5847 NOREF(pMixedCtx);
5848 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5849 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5850 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5851 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5852}
5853
5854
5855/**
5856 * Handle a condition that occurred while delivering an event through the guest
5857 * IDT.
5858 *
5859 * @returns Strict VBox status code (i.e. informational status codes too).
5860 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5861 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5862 * to continue execution of the guest which will delivery the \#DF.
5863 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5864 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5865 *
5866 * @param pVCpu The cross context virtual CPU structure.
5867 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5868 * out-of-sync. Make sure to update the required fields
5869 * before using them.
5870 * @param pVmxTransient Pointer to the VMX transient structure.
5871 *
5872 * @remarks No-long-jump zone!!!
5873 */
5874static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5875{
5876 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5877
5878 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5879 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5880 AssertRCReturn(rc2, rc2);
5881
5882 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5883 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5884 {
5885 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5886 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5887
5888 /*
5889 * If the event was a software interrupt (generated with INT n) or a software exception
5890 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5891 * can handle the VM-exit and continue guest execution which will re-execute the
5892 * instruction rather than re-injecting the exception, as that can cause premature
5893 * trips to ring-3 before injection and involve TRPM which currently has no way of
5894 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5895 * the problem).
5896 */
5897 IEMXCPTRAISE enmRaise;
5898 IEMXCPTRAISEINFO fRaiseInfo;
5899 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5900 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5901 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5902 {
5903 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5904 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5905 }
5906 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5907 {
5908 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5909 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5910 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5911 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5912 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5913 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5914 uExitVectorType), VERR_VMX_IPE_5);
5915
5916 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5917
5918 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5919 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5920 {
5921 pVmxTransient->fVectoringPF = true;
5922 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5923 }
5924 }
5925 else
5926 {
5927 /*
5928 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5929 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5930 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5931 */
5932 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5933 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5934 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5935 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5936 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5937 }
5938
5939 /*
5940 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5941 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5942 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5943 * subsequent VM-entry would fail.
5944 *
5945 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5946 */
5947 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5948 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5949 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5950 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5951 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5952 {
5953 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5954 }
5955
5956 switch (enmRaise)
5957 {
5958 case IEMXCPTRAISE_CURRENT_XCPT:
5959 {
5960 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5961 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5962 Assert(rcStrict == VINF_SUCCESS);
5963 break;
5964 }
5965
5966 case IEMXCPTRAISE_PREV_EVENT:
5967 {
5968 uint32_t u32ErrCode;
5969 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5970 {
5971 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5972 AssertRCReturn(rc2, rc2);
5973 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5974 }
5975 else
5976 u32ErrCode = 0;
5977
5978 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5979 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5980 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5981 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5982
5983 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5984 pVCpu->hm.s.Event.u32ErrCode));
5985 Assert(rcStrict == VINF_SUCCESS);
5986 break;
5987 }
5988
5989 case IEMXCPTRAISE_REEXEC_INSTR:
5990 Assert(rcStrict == VINF_SUCCESS);
5991 break;
5992
5993 case IEMXCPTRAISE_DOUBLE_FAULT:
5994 {
5995 /*
5996 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5997 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5998 */
5999 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
6000 {
6001 pVmxTransient->fVectoringDoublePF = true;
6002 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
6003 pMixedCtx->cr2));
6004 rcStrict = VINF_SUCCESS;
6005 }
6006 else
6007 {
6008 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
6009 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
6010 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
6011 uIdtVector, uExitVector));
6012 rcStrict = VINF_HM_DOUBLE_FAULT;
6013 }
6014 break;
6015 }
6016
6017 case IEMXCPTRAISE_TRIPLE_FAULT:
6018 {
6019 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
6020 rcStrict = VINF_EM_RESET;
6021 break;
6022 }
6023
6024 case IEMXCPTRAISE_CPU_HANG:
6025 {
6026 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6027 rcStrict = VERR_EM_GUEST_CPU_HANG;
6028 break;
6029 }
6030
6031 default:
6032 {
6033 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6034 rcStrict = VERR_VMX_IPE_2;
6035 break;
6036 }
6037 }
6038 }
6039 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6040 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6041 && uExitVector != X86_XCPT_DF
6042 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6043 {
6044 /*
6045 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6046 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6047 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6048 */
6049 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6050 {
6051 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6052 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6053 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6054 }
6055 }
6056
6057 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6058 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6059 return rcStrict;
6060}
6061
6062
6063/**
6064 * Imports a guest segment register from the current VMCS into
6065 * the guest-CPU context.
6066 *
6067 * @returns VBox status code.
6068 * @param pVCpu The cross context virtual CPU structure.
6069 * @param idxSel Index of the selector in the VMCS.
6070 * @param idxLimit Index of the segment limit in the VMCS.
6071 * @param idxBase Index of the segment base in the VMCS.
6072 * @param idxAccess Index of the access rights of the segment in the VMCS.
6073 * @param pSelReg Pointer to the segment selector.
6074 *
6075 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6076 * do not log!
6077 *
6078 * @remarks Never call this function directly!!! Use the
6079 * HMVMX_IMPORT_SREG() macro as that takes care
6080 * of whether to read from the VMCS cache or not.
6081 */
6082static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6083 PCPUMSELREG pSelReg)
6084{
6085 NOREF(pVCpu);
6086
6087 uint32_t u32Sel;
6088 uint32_t u32Limit;
6089 uint32_t u32Attr;
6090 uint64_t u64Base;
6091 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6092 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6093 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6094 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6095 AssertRCReturn(rc, rc);
6096
6097 pSelReg->Sel = (uint16_t)u32Sel;
6098 pSelReg->ValidSel = (uint16_t)u32Sel;
6099 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6100 pSelReg->u32Limit = u32Limit;
6101 pSelReg->u64Base = u64Base;
6102 pSelReg->Attr.u = u32Attr;
6103
6104 /*
6105 * If VT-x marks the segment as unusable, most other bits remain undefined:
6106 * - For CS the L, D and G bits have meaning.
6107 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6108 * - For the remaining data segments no bits are defined.
6109 *
6110 * The present bit and the unusable bit has been observed to be set at the
6111 * same time (the selector was supposed to be invalid as we started executing
6112 * a V8086 interrupt in ring-0).
6113 *
6114 * What should be important for the rest of the VBox code, is that the P bit is
6115 * cleared. Some of the other VBox code recognizes the unusable bit, but
6116 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6117 * safe side here, we'll strip off P and other bits we don't care about. If
6118 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6119 *
6120 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6121 */
6122 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6123 {
6124 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6125
6126 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6127 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6128 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6129#ifdef VBOX_STRICT
6130 VMMRZCallRing3Disable(pVCpu);
6131 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6132# ifdef DEBUG_bird
6133 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6134 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6135 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6136# endif
6137 VMMRZCallRing3Enable(pVCpu);
6138#endif
6139 }
6140 return VINF_SUCCESS;
6141}
6142
6143
6144/**
6145 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6146 *
6147 * @returns VBox status code.
6148 * @param pVCpu The cross context virtual CPU structure.
6149 *
6150 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6151 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6152 * instead!!!
6153 */
6154DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6155{
6156 uint64_t u64Val;
6157 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6158 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6159 {
6160 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6161 if (RT_SUCCESS(rc))
6162 {
6163 pCtx->rip = u64Val;
6164 EMR0HistoryUpdatePC(pVCpu, pCtx->rip, false);
6165 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6166 }
6167 return rc;
6168 }
6169 return VINF_SUCCESS;
6170}
6171
6172
6173/**
6174 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6175 *
6176 * @returns VBox status code.
6177 * @param pVCpu The cross context virtual CPU structure.
6178 *
6179 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6180 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6181 * instead!!!
6182 */
6183DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6184{
6185 uint32_t u32Val;
6186 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6187 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6188 {
6189 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6190 if (RT_SUCCESS(rc))
6191 {
6192 pCtx->eflags.u32 = u32Val;
6193
6194 /* Restore eflags for real-on-v86-mode hack. */
6195 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6196 {
6197 pCtx->eflags.Bits.u1VM = 0;
6198 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6199 }
6200 }
6201 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6202 return rc;
6203 }
6204 return VINF_SUCCESS;
6205}
6206
6207
6208/**
6209 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6210 * context.
6211 *
6212 * @returns VBox status code.
6213 * @param pVCpu The cross context virtual CPU structure.
6214 *
6215 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6216 * do not log!
6217 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6218 * instead!!!
6219 */
6220DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6221{
6222 uint32_t u32Val;
6223 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6224 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6225 if (RT_SUCCESS(rc))
6226 {
6227 /*
6228 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6229 * might need them in hmR0VmxEvaluatePendingEvent().
6230 */
6231 if (!u32Val)
6232 {
6233 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6234 {
6235 rc = hmR0VmxImportGuestRip(pVCpu);
6236 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6237 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6238 }
6239
6240 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6241 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6242 }
6243 else
6244 {
6245 rc = hmR0VmxImportGuestRip(pVCpu);
6246 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6247
6248 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6249 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6250 {
6251 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6252 }
6253 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6254 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6255
6256 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6257 {
6258 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6259 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6260 }
6261 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6262 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6263 }
6264 }
6265 return rc;
6266}
6267
6268
6269/**
6270 * Worker for VMXR0ImportStateOnDemand.
6271 *
6272 * @returns VBox status code.
6273 * @param pVCpu The cross context virtual CPU structure.
6274 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6275 */
6276static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6277{
6278#define VMXLOCAL_BREAK_RC(a_rc) \
6279 if (RT_FAILURE(a_rc)) \
6280 break
6281
6282 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6283
6284 int rc = VINF_SUCCESS;
6285 PVM pVM = pVCpu->CTX_SUFF(pVM);
6286 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6287 uint64_t u64Val;
6288 uint32_t u32Val;
6289
6290 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6291
6292 /*
6293 * We disable interrupts to make the updating of the state and in particular
6294 * the fExtrn modification atomic wrt to preemption hooks.
6295 */
6296 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6297
6298 fWhat &= pCtx->fExtrn;
6299 if (fWhat)
6300 {
6301 do
6302 {
6303 if (fWhat & CPUMCTX_EXTRN_RIP)
6304 {
6305 rc = hmR0VmxImportGuestRip(pVCpu);
6306 VMXLOCAL_BREAK_RC(rc);
6307 }
6308
6309 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6310 {
6311 rc = hmR0VmxImportGuestRFlags(pVCpu);
6312 VMXLOCAL_BREAK_RC(rc);
6313 }
6314
6315 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6316 {
6317 rc = hmR0VmxImportGuestIntrState(pVCpu);
6318 VMXLOCAL_BREAK_RC(rc);
6319 }
6320
6321 if (fWhat & CPUMCTX_EXTRN_RSP)
6322 {
6323 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6324 VMXLOCAL_BREAK_RC(rc);
6325 pCtx->rsp = u64Val;
6326 }
6327
6328 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6329 {
6330 if (fWhat & CPUMCTX_EXTRN_CS)
6331 {
6332 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6333 rc |= hmR0VmxImportGuestRip(pVCpu);
6334 VMXLOCAL_BREAK_RC(rc);
6335 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6336 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6337 EMR0HistoryUpdatePC(pVCpu, pCtx->cs.u64Base + pCtx->rip, true);
6338 }
6339 if (fWhat & CPUMCTX_EXTRN_SS)
6340 {
6341 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6342 VMXLOCAL_BREAK_RC(rc);
6343 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6344 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6345 }
6346 if (fWhat & CPUMCTX_EXTRN_DS)
6347 {
6348 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6349 VMXLOCAL_BREAK_RC(rc);
6350 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6351 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6352 }
6353 if (fWhat & CPUMCTX_EXTRN_ES)
6354 {
6355 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6356 VMXLOCAL_BREAK_RC(rc);
6357 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6358 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6359 }
6360 if (fWhat & CPUMCTX_EXTRN_FS)
6361 {
6362 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6363 VMXLOCAL_BREAK_RC(rc);
6364 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6365 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6366 }
6367 if (fWhat & CPUMCTX_EXTRN_GS)
6368 {
6369 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6370 VMXLOCAL_BREAK_RC(rc);
6371 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6372 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6373 }
6374 }
6375
6376 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6377 {
6378 if (fWhat & CPUMCTX_EXTRN_LDTR)
6379 {
6380 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6381 VMXLOCAL_BREAK_RC(rc);
6382 }
6383
6384 if (fWhat & CPUMCTX_EXTRN_GDTR)
6385 {
6386 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6387 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6388 VMXLOCAL_BREAK_RC(rc);
6389 pCtx->gdtr.pGdt = u64Val;
6390 pCtx->gdtr.cbGdt = u32Val;
6391 }
6392
6393 /* Guest IDTR. */
6394 if (fWhat & CPUMCTX_EXTRN_IDTR)
6395 {
6396 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6397 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6398 VMXLOCAL_BREAK_RC(rc);
6399 pCtx->idtr.pIdt = u64Val;
6400 pCtx->idtr.cbIdt = u32Val;
6401 }
6402
6403 /* Guest TR. */
6404 if (fWhat & CPUMCTX_EXTRN_TR)
6405 {
6406 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6407 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6408 {
6409 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6410 VMXLOCAL_BREAK_RC(rc);
6411 }
6412 }
6413 }
6414
6415 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6416 {
6417 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6418 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6419 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6420 pCtx->SysEnter.cs = u32Val;
6421 VMXLOCAL_BREAK_RC(rc);
6422 }
6423
6424#if HC_ARCH_BITS == 64
6425 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6426 {
6427 if ( pVM->hm.s.fAllow64BitGuests
6428 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6429 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6430 }
6431
6432 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6433 {
6434 if ( pVM->hm.s.fAllow64BitGuests
6435 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6436 {
6437 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6438 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6439 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6440 }
6441 }
6442#endif
6443
6444 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6445#if HC_ARCH_BITS == 32
6446 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6447#endif
6448 )
6449 {
6450 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6451 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6452 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6453 {
6454 switch (pMsr->u32Msr)
6455 {
6456#if HC_ARCH_BITS == 32
6457 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6458 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6459 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6460 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6461#endif
6462 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6463 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6464 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6465 default:
6466 {
6467 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6468 ASMSetFlags(fEFlags);
6469 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6470 cMsrs));
6471 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6472 }
6473 }
6474 }
6475 }
6476
6477 if (fWhat & CPUMCTX_EXTRN_DR7)
6478 {
6479 if (!pVCpu->hm.s.fUsingHyperDR7)
6480 {
6481 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6482 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6483 VMXLOCAL_BREAK_RC(rc);
6484 pCtx->dr[7] = u32Val;
6485 }
6486 }
6487
6488 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6489 {
6490 uint32_t u32Shadow;
6491 if (fWhat & CPUMCTX_EXTRN_CR0)
6492 {
6493 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6494 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6495 VMXLOCAL_BREAK_RC(rc);
6496 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6497 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6498 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6499 CPUMSetGuestCR0(pVCpu, u32Val);
6500 VMMRZCallRing3Enable(pVCpu);
6501 }
6502
6503 if (fWhat & CPUMCTX_EXTRN_CR4)
6504 {
6505 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6506 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6507 VMXLOCAL_BREAK_RC(rc);
6508 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6509 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6510 CPUMSetGuestCR4(pVCpu, u32Val);
6511 }
6512
6513 if (fWhat & CPUMCTX_EXTRN_CR3)
6514 {
6515 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6516 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6517 || ( pVM->hm.s.fNestedPaging
6518 && CPUMIsGuestPagingEnabledEx(pCtx)))
6519 {
6520 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6521 if (pCtx->cr3 != u64Val)
6522 {
6523 CPUMSetGuestCR3(pVCpu, u64Val);
6524 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6525 }
6526
6527 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6528 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6529 if (CPUMIsGuestInPAEModeEx(pCtx))
6530 {
6531 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6532 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6533 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6534 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6535 VMXLOCAL_BREAK_RC(rc);
6536 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6537 }
6538 }
6539 }
6540 }
6541 } while (0);
6542
6543 if (RT_SUCCESS(rc))
6544 {
6545 /* Update fExtrn. */
6546 pCtx->fExtrn &= ~fWhat;
6547
6548 /* If everything has been imported, clear the HM keeper bit. */
6549 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6550 {
6551 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6552 Assert(!pCtx->fExtrn);
6553 }
6554 }
6555 }
6556 else
6557 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6558
6559 ASMSetFlags(fEFlags);
6560
6561 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6562
6563 /*
6564 * Honor any pending CR3 updates.
6565 *
6566 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6567 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6568 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6569 *
6570 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6571 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6572 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6573 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6574 *
6575 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6576 */
6577 if (VMMRZCallRing3IsEnabled(pVCpu))
6578 {
6579 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6580 {
6581 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6582 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6583 }
6584
6585 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6586 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6587
6588 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6589 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6590 }
6591
6592 return VINF_SUCCESS;
6593#undef VMXLOCAL_BREAK_RC
6594}
6595
6596
6597/**
6598 * Saves the guest state from the VMCS into the guest-CPU context.
6599 *
6600 * @returns VBox status code.
6601 * @param pVCpu The cross context virtual CPU structure.
6602 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6603 */
6604VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6605{
6606 return hmR0VmxImportGuestState(pVCpu, fWhat);
6607}
6608
6609
6610/**
6611 * Check per-VM and per-VCPU force flag actions that require us to go back to
6612 * ring-3 for one reason or another.
6613 *
6614 * @returns Strict VBox status code (i.e. informational status codes too)
6615 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6616 * ring-3.
6617 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6618 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6619 * interrupts)
6620 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6621 * all EMTs to be in ring-3.
6622 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6623 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6624 * to the EM loop.
6625 *
6626 * @param pVCpu The cross context virtual CPU structure.
6627 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6628 * out-of-sync. Make sure to update the required fields
6629 * before using them.
6630 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6631 */
6632static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6633{
6634 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6635
6636 /*
6637 * Anything pending? Should be more likely than not if we're doing a good job.
6638 */
6639 PVM pVM = pVCpu->CTX_SUFF(pVM);
6640 if ( !fStepping
6641 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6642 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6643 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6644 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6645 return VINF_SUCCESS;
6646
6647 /* Pending PGM C3 sync. */
6648 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6649 {
6650 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6651 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6652 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6653 if (rcStrict2 != VINF_SUCCESS)
6654 {
6655 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6656 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6657 return rcStrict2;
6658 }
6659 }
6660
6661 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6662 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6663 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6664 {
6665 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6666 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6667 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6668 return rc2;
6669 }
6670
6671 /* Pending VM request packets, such as hardware interrupts. */
6672 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6673 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6674 {
6675 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6676 return VINF_EM_PENDING_REQUEST;
6677 }
6678
6679 /* Pending PGM pool flushes. */
6680 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6681 {
6682 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6683 return VINF_PGM_POOL_FLUSH_PENDING;
6684 }
6685
6686 /* Pending DMA requests. */
6687 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6688 {
6689 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6690 return VINF_EM_RAW_TO_R3;
6691 }
6692
6693 return VINF_SUCCESS;
6694}
6695
6696
6697/**
6698 * Converts any TRPM trap into a pending HM event. This is typically used when
6699 * entering from ring-3 (not longjmp returns).
6700 *
6701 * @param pVCpu The cross context virtual CPU structure.
6702 */
6703static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6704{
6705 Assert(TRPMHasTrap(pVCpu));
6706 Assert(!pVCpu->hm.s.Event.fPending);
6707
6708 uint8_t uVector;
6709 TRPMEVENT enmTrpmEvent;
6710 RTGCUINT uErrCode;
6711 RTGCUINTPTR GCPtrFaultAddress;
6712 uint8_t cbInstr;
6713
6714 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6715 AssertRC(rc);
6716
6717 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6718 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6719 if (enmTrpmEvent == TRPM_TRAP)
6720 {
6721 switch (uVector)
6722 {
6723 case X86_XCPT_NMI:
6724 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6725 break;
6726
6727 case X86_XCPT_BP:
6728 case X86_XCPT_OF:
6729 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6730 break;
6731
6732 case X86_XCPT_PF:
6733 case X86_XCPT_DF:
6734 case X86_XCPT_TS:
6735 case X86_XCPT_NP:
6736 case X86_XCPT_SS:
6737 case X86_XCPT_GP:
6738 case X86_XCPT_AC:
6739 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6740 RT_FALL_THRU();
6741 default:
6742 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6743 break;
6744 }
6745 }
6746 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6747 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6748 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6749 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6750 else
6751 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6752
6753 rc = TRPMResetTrap(pVCpu);
6754 AssertRC(rc);
6755 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6756 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6757
6758 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6759}
6760
6761
6762/**
6763 * Converts the pending HM event into a TRPM trap.
6764 *
6765 * @param pVCpu The cross context virtual CPU structure.
6766 */
6767static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6768{
6769 Assert(pVCpu->hm.s.Event.fPending);
6770
6771 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6772 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6773 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6774 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6775
6776 /* If a trap was already pending, we did something wrong! */
6777 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6778
6779 TRPMEVENT enmTrapType;
6780 switch (uVectorType)
6781 {
6782 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6783 enmTrapType = TRPM_HARDWARE_INT;
6784 break;
6785
6786 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6787 enmTrapType = TRPM_SOFTWARE_INT;
6788 break;
6789
6790 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6791 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6792 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6793 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6794 enmTrapType = TRPM_TRAP;
6795 break;
6796
6797 default:
6798 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6799 enmTrapType = TRPM_32BIT_HACK;
6800 break;
6801 }
6802
6803 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6804
6805 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6806 AssertRC(rc);
6807
6808 if (fErrorCodeValid)
6809 TRPMSetErrorCode(pVCpu, uErrorCode);
6810
6811 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6812 && uVector == X86_XCPT_PF)
6813 {
6814 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6815 }
6816 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6817 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6818 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6819 {
6820 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6821 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6822 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6823 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6824 }
6825
6826 /* Clear any pending events from the VMCS. */
6827 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6828 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6829
6830 /* We're now done converting the pending event. */
6831 pVCpu->hm.s.Event.fPending = false;
6832}
6833
6834
6835/**
6836 * Does the necessary state syncing before returning to ring-3 for any reason
6837 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6838 *
6839 * @returns VBox status code.
6840 * @param pVCpu The cross context virtual CPU structure.
6841 * @param fImportState Whether to import the guest state from the VMCS back
6842 * to the guest-CPU context.
6843 *
6844 * @remarks No-long-jmp zone!!!
6845 */
6846static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6847{
6848 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6849 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6850
6851 RTCPUID idCpu = RTMpCpuId();
6852 Log4Func(("HostCpuId=%u\n", idCpu));
6853
6854 /*
6855 * !!! IMPORTANT !!!
6856 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6857 */
6858
6859 /* Save the guest state if necessary. */
6860 if (fImportState)
6861 {
6862 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6863 AssertRCReturn(rc, rc);
6864 }
6865
6866 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6867 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6868 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6869
6870 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6871#ifdef VBOX_STRICT
6872 if (CPUMIsHyperDebugStateActive(pVCpu))
6873 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6874#endif
6875 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6876 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6877 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6878
6879#if HC_ARCH_BITS == 64
6880 /* Restore host-state bits that VT-x only restores partially. */
6881 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6882 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6883 {
6884 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6885 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6886 }
6887 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6888#endif
6889
6890 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6891 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6892 {
6893 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6894 if (!fImportState)
6895 {
6896 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6897 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6898 AssertRCReturn(rc, rc);
6899 }
6900 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6901 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6902 }
6903 else
6904 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6905
6906 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6907 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6908
6909 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6910 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6911 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6912 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6913 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6914 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6915 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6916 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6917 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6918
6919 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6920
6921 /** @todo This partially defeats the purpose of having preemption hooks.
6922 * The problem is, deregistering the hooks should be moved to a place that
6923 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6924 * context.
6925 */
6926 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6927 {
6928 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6929 AssertRCReturn(rc, rc);
6930
6931 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6932 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6933 }
6934 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6935 NOREF(idCpu);
6936
6937 return VINF_SUCCESS;
6938}
6939
6940
6941/**
6942 * Leaves the VT-x session.
6943 *
6944 * @returns VBox status code.
6945 * @param pVCpu The cross context virtual CPU structure.
6946 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6947 * out-of-sync. Make sure to update the required fields
6948 * before using them.
6949 *
6950 * @remarks No-long-jmp zone!!!
6951 */
6952static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6953{
6954 HM_DISABLE_PREEMPT();
6955 HMVMX_ASSERT_CPU_SAFE();
6956 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6957 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6958
6959 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6960 and done this from the VMXR0ThreadCtxCallback(). */
6961 if (!pVCpu->hm.s.fLeaveDone)
6962 {
6963 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6964 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6965 pVCpu->hm.s.fLeaveDone = true;
6966 }
6967 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6968
6969 /*
6970 * !!! IMPORTANT !!!
6971 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6972 */
6973
6974 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6975 /** @todo Deregistering here means we need to VMCLEAR always
6976 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6977 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6978 VMMR0ThreadCtxHookDisable(pVCpu);
6979
6980 /* Leave HM context. This takes care of local init (term). */
6981 int rc = HMR0LeaveCpu(pVCpu);
6982
6983 HM_RESTORE_PREEMPT();
6984 return rc;
6985}
6986
6987
6988/**
6989 * Does the necessary state syncing before doing a longjmp to ring-3.
6990 *
6991 * @returns VBox status code.
6992 * @param pVCpu The cross context virtual CPU structure.
6993 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6994 * out-of-sync. Make sure to update the required fields
6995 * before using them.
6996 *
6997 * @remarks No-long-jmp zone!!!
6998 */
6999DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7000{
7001 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7002}
7003
7004
7005/**
7006 * Take necessary actions before going back to ring-3.
7007 *
7008 * An action requires us to go back to ring-3. This function does the necessary
7009 * steps before we can safely return to ring-3. This is not the same as longjmps
7010 * to ring-3, this is voluntary and prepares the guest so it may continue
7011 * executing outside HM (recompiler/IEM).
7012 *
7013 * @returns VBox status code.
7014 * @param pVCpu The cross context virtual CPU structure.
7015 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7016 * out-of-sync. Make sure to update the required fields
7017 * before using them.
7018 * @param rcExit The reason for exiting to ring-3. Can be
7019 * VINF_VMM_UNKNOWN_RING3_CALL.
7020 */
7021static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
7022{
7023 Assert(pVCpu);
7024 Assert(pMixedCtx);
7025 HMVMX_ASSERT_PREEMPT_SAFE();
7026
7027 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7028 {
7029 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7030 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7031 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7032 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7033 }
7034
7035 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7036 VMMRZCallRing3Disable(pVCpu);
7037 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7038
7039 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7040 if (pVCpu->hm.s.Event.fPending)
7041 {
7042 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7043 Assert(!pVCpu->hm.s.Event.fPending);
7044 }
7045
7046 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7047 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7048
7049 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7050 and if we're injecting an event we should have a TRPM trap pending. */
7051 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7052#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7053 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7054#endif
7055
7056 /* Save guest state and restore host state bits. */
7057 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7058 AssertRCReturn(rc, rc);
7059 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7060 /* Thread-context hooks are unregistered at this point!!! */
7061
7062 /* Sync recompiler state. */
7063 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7064 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7065 | CPUM_CHANGED_LDTR
7066 | CPUM_CHANGED_GDTR
7067 | CPUM_CHANGED_IDTR
7068 | CPUM_CHANGED_TR
7069 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7070 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7071 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7072 {
7073 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7074 }
7075
7076 Assert(!pVCpu->hm.s.fClearTrapFlag);
7077
7078 /* Update the exit-to-ring 3 reason. */
7079 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7080
7081 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7082 if (rcExit != VINF_EM_RAW_INTERRUPT)
7083 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7084
7085 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7086
7087 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7088 VMMRZCallRing3RemoveNotification(pVCpu);
7089 VMMRZCallRing3Enable(pVCpu);
7090
7091 return rc;
7092}
7093
7094
7095/**
7096 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7097 * longjump to ring-3 and possibly get preempted.
7098 *
7099 * @returns VBox status code.
7100 * @param pVCpu The cross context virtual CPU structure.
7101 * @param enmOperation The operation causing the ring-3 longjump.
7102 * @param pvUser Opaque pointer to the guest-CPU context. The data
7103 * may be out-of-sync. Make sure to update the required
7104 * fields before using them.
7105 */
7106static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7107{
7108 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7109 {
7110 /*
7111 * !!! IMPORTANT !!!
7112 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7113 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7114 */
7115 VMMRZCallRing3RemoveNotification(pVCpu);
7116 VMMRZCallRing3Disable(pVCpu);
7117 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7118 RTThreadPreemptDisable(&PreemptState);
7119
7120 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7121 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7122
7123#if HC_ARCH_BITS == 64
7124 /* Restore host-state bits that VT-x only restores partially. */
7125 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7126 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7127 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7128 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7129#endif
7130
7131 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7132 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7133 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7134
7135 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7136 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7137 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7138 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7139 {
7140 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7141 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7142 }
7143
7144 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7145 VMMR0ThreadCtxHookDisable(pVCpu);
7146 HMR0LeaveCpu(pVCpu);
7147 RTThreadPreemptRestore(&PreemptState);
7148 return VINF_SUCCESS;
7149 }
7150
7151 Assert(pVCpu);
7152 Assert(pvUser);
7153 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7154 HMVMX_ASSERT_PREEMPT_SAFE();
7155
7156 VMMRZCallRing3Disable(pVCpu);
7157 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7158
7159 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7160
7161 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7162 AssertRCReturn(rc, rc);
7163
7164 VMMRZCallRing3Enable(pVCpu);
7165 return VINF_SUCCESS;
7166}
7167
7168
7169/**
7170 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7171 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7172 *
7173 * @param pVCpu The cross context virtual CPU structure.
7174 */
7175DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7176{
7177 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7178 {
7179 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7180 {
7181 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7182 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7183 AssertRC(rc);
7184 Log4Func(("Setup interrupt-window exiting\n"));
7185 }
7186 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7187}
7188
7189
7190/**
7191 * Clears the interrupt-window exiting control in the VMCS.
7192 *
7193 * @param pVCpu The cross context virtual CPU structure.
7194 */
7195DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7196{
7197 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7198 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7199 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7200 AssertRC(rc);
7201 Log4Func(("Cleared interrupt-window exiting\n"));
7202}
7203
7204
7205/**
7206 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7207 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7208 *
7209 * @param pVCpu The cross context virtual CPU structure.
7210 */
7211DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7212{
7213 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7214 {
7215 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7216 {
7217 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7218 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7219 AssertRC(rc);
7220 Log4Func(("Setup NMI-window exiting\n"));
7221 }
7222 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7223}
7224
7225
7226/**
7227 * Clears the NMI-window exiting control in the VMCS.
7228 *
7229 * @param pVCpu The cross context virtual CPU structure.
7230 */
7231DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7232{
7233 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7234 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7235 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7236 AssertRC(rc);
7237 Log4Func(("Cleared NMI-window exiting\n"));
7238}
7239
7240
7241/**
7242 * Evaluates the event to be delivered to the guest and sets it as the pending
7243 * event.
7244 *
7245 * @returns The VT-x guest-interruptibility state.
7246 * @param pVCpu The cross context virtual CPU structure.
7247 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7248 * out-of-sync. Make sure to update the required fields
7249 * before using them.
7250 */
7251static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7252{
7253 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7254 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7255 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7256 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7257 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7258
7259 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7260 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7261 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7262 Assert(!TRPMHasTrap(pVCpu));
7263
7264 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7265 APICUpdatePendingInterrupts(pVCpu);
7266
7267 /*
7268 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7269 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7270 */
7271 /** @todo SMI. SMIs take priority over NMIs. */
7272 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7273 {
7274 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7275 if ( !pVCpu->hm.s.Event.fPending
7276 && !fBlockNmi
7277 && !fBlockSti
7278 && !fBlockMovSS)
7279 {
7280 Log4Func(("Pending NMI\n"));
7281 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7282 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7283
7284 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7285 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7286 }
7287 else
7288 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7289 }
7290 /*
7291 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7292 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7293 */
7294 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7295 && !pVCpu->hm.s.fSingleInstruction)
7296 {
7297 Assert(!DBGFIsStepping(pVCpu));
7298 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7299 AssertRCReturn(rc, 0);
7300 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7301 if ( !pVCpu->hm.s.Event.fPending
7302 && !fBlockInt
7303 && !fBlockSti
7304 && !fBlockMovSS)
7305 {
7306 uint8_t u8Interrupt;
7307 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7308 if (RT_SUCCESS(rc))
7309 {
7310 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7311 uint32_t u32IntInfo = u8Interrupt
7312 | VMX_EXIT_INTERRUPTION_INFO_VALID
7313 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7314
7315 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7316 }
7317 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7318 {
7319 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7320 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7321 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7322
7323 /*
7324 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7325 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7326 * need to re-set this force-flag here.
7327 */
7328 }
7329 else
7330 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7331 }
7332 else
7333 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7334 }
7335
7336 return fIntrState;
7337}
7338
7339
7340/**
7341 * Sets a pending-debug exception to be delivered to the guest if the guest is
7342 * single-stepping in the VMCS.
7343 *
7344 * @param pVCpu The cross context virtual CPU structure.
7345 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7346 * out-of-sync. Make sure to update the required fields
7347 * before using them.
7348 */
7349DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7350{
7351 RT_NOREF(pVCpu);
7352 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7353 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7354}
7355
7356
7357/**
7358 * Injects any pending events into the guest if the guest is in a state to
7359 * receive them.
7360 *
7361 * @returns Strict VBox status code (i.e. informational status codes too).
7362 * @param pVCpu The cross context virtual CPU structure.
7363 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7364 * out-of-sync. Make sure to update the required fields
7365 * before using them.
7366 * @param fIntrState The VT-x guest-interruptibility state.
7367 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7368 * return VINF_EM_DBG_STEPPED if the event was
7369 * dispatched directly.
7370 */
7371static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7372{
7373 HMVMX_ASSERT_PREEMPT_SAFE();
7374 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7375
7376 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7377 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7378
7379 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7380 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7381 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7382 Assert(!TRPMHasTrap(pVCpu));
7383
7384 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7385 if (pVCpu->hm.s.Event.fPending)
7386 {
7387 /*
7388 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7389 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7390 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7391 *
7392 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7393 */
7394 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7395#ifdef VBOX_STRICT
7396 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7397 {
7398 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7399 Assert(!fBlockInt);
7400 Assert(!fBlockSti);
7401 Assert(!fBlockMovSS);
7402 }
7403 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7404 {
7405 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7406 Assert(!fBlockSti);
7407 Assert(!fBlockMovSS);
7408 Assert(!fBlockNmi);
7409 }
7410#endif
7411 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7412 uIntType));
7413 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7414 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7415 &fIntrState);
7416 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7417
7418 /* Update the interruptibility-state as it could have been changed by
7419 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7420 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7421 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7422
7423 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7424 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7425 else
7426 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7427 }
7428
7429 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7430 if ( fBlockSti
7431 || fBlockMovSS)
7432 {
7433 if (!pVCpu->hm.s.fSingleInstruction)
7434 {
7435 /*
7436 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7437 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7438 * See Intel spec. 27.3.4 "Saving Non-Register State".
7439 */
7440 Assert(!DBGFIsStepping(pVCpu));
7441 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7442 AssertRCReturn(rc, rc);
7443 if (pMixedCtx->eflags.Bits.u1TF)
7444 {
7445 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7446 AssertRCReturn(rc2, rc2);
7447 }
7448 }
7449 else if (pMixedCtx->eflags.Bits.u1TF)
7450 {
7451 /*
7452 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7453 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7454 */
7455 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7456 fIntrState = 0;
7457 }
7458 }
7459
7460 /*
7461 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7462 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7463 */
7464 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7465 AssertRCReturn(rc3, rc3);
7466
7467 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7468 NOREF(fBlockMovSS); NOREF(fBlockSti);
7469 return rcStrict;
7470}
7471
7472
7473/**
7474 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7475 *
7476 * @param pVCpu The cross context virtual CPU structure.
7477 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7478 * out-of-sync. Make sure to update the required fields
7479 * before using them.
7480 */
7481DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7482{
7483 NOREF(pMixedCtx);
7484 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7485 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7486}
7487
7488
7489/**
7490 * Injects a double-fault (\#DF) exception into the VM.
7491 *
7492 * @returns Strict VBox status code (i.e. informational status codes too).
7493 * @param pVCpu The cross context virtual CPU structure.
7494 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7495 * out-of-sync. Make sure to update the required fields
7496 * before using them.
7497 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7498 * and should return VINF_EM_DBG_STEPPED if the event
7499 * is injected directly (register modified by us, not
7500 * by hardware on VM-entry).
7501 * @param pfIntrState Pointer to the current guest interruptibility-state.
7502 * This interruptibility-state will be updated if
7503 * necessary. This cannot not be NULL.
7504 */
7505DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7506{
7507 NOREF(pMixedCtx);
7508 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7509 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7510 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7511 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7512 pfIntrState);
7513}
7514
7515
7516/**
7517 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7518 *
7519 * @param pVCpu The cross context virtual CPU structure.
7520 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7521 * out-of-sync. Make sure to update the required fields
7522 * before using them.
7523 */
7524DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7525{
7526 NOREF(pMixedCtx);
7527 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7528 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7529 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7530}
7531
7532
7533/**
7534 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7535 *
7536 * @param pVCpu The cross context virtual CPU structure.
7537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7538 * out-of-sync. Make sure to update the required fields
7539 * before using them.
7540 * @param cbInstr The value of RIP that is to be pushed on the guest
7541 * stack.
7542 */
7543DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7544{
7545 NOREF(pMixedCtx);
7546 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7547 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7548 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7549}
7550
7551
7552/**
7553 * Injects a general-protection (\#GP) fault into the VM.
7554 *
7555 * @returns Strict VBox status code (i.e. informational status codes too).
7556 * @param pVCpu The cross context virtual CPU structure.
7557 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7558 * out-of-sync. Make sure to update the required fields
7559 * before using them.
7560 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7561 * mode, i.e. in real-mode it's not valid).
7562 * @param u32ErrorCode The error code associated with the \#GP.
7563 * @param fStepping Whether we're running in
7564 * hmR0VmxRunGuestCodeStep() and should return
7565 * VINF_EM_DBG_STEPPED if the event is injected
7566 * directly (register modified by us, not by
7567 * hardware on VM-entry).
7568 * @param pfIntrState Pointer to the current guest interruptibility-state.
7569 * This interruptibility-state will be updated if
7570 * necessary. This cannot not be NULL.
7571 */
7572DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7573 bool fStepping, uint32_t *pfIntrState)
7574{
7575 NOREF(pMixedCtx);
7576 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7577 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7578 if (fErrorCodeValid)
7579 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7580 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7581 pfIntrState);
7582}
7583
7584
7585#if 0 /* unused */
7586/**
7587 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7588 * VM.
7589 *
7590 * @param pVCpu The cross context virtual CPU structure.
7591 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7592 * out-of-sync. Make sure to update the required fields
7593 * before using them.
7594 * @param u32ErrorCode The error code associated with the \#GP.
7595 */
7596DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7597{
7598 NOREF(pMixedCtx);
7599 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7600 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7601 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7602 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7603}
7604#endif /* unused */
7605
7606
7607/**
7608 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7609 *
7610 * @param pVCpu The cross context virtual CPU structure.
7611 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7612 * out-of-sync. Make sure to update the required fields
7613 * before using them.
7614 * @param uVector The software interrupt vector number.
7615 * @param cbInstr The value of RIP that is to be pushed on the guest
7616 * stack.
7617 */
7618DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7619{
7620 NOREF(pMixedCtx);
7621 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7622 if ( uVector == X86_XCPT_BP
7623 || uVector == X86_XCPT_OF)
7624 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7625 else
7626 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7627 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7628}
7629
7630
7631/**
7632 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7633 * stack.
7634 *
7635 * @returns Strict VBox status code (i.e. informational status codes too).
7636 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7637 * @param pVM The cross context VM structure.
7638 * @param pMixedCtx Pointer to the guest-CPU context.
7639 * @param uValue The value to push to the guest stack.
7640 */
7641DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7642{
7643 /*
7644 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7645 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7646 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7647 */
7648 if (pMixedCtx->sp == 1)
7649 return VINF_EM_RESET;
7650 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7651 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7652 AssertRC(rc);
7653 return rc;
7654}
7655
7656
7657/**
7658 * Injects an event into the guest upon VM-entry by updating the relevant fields
7659 * in the VM-entry area in the VMCS.
7660 *
7661 * @returns Strict VBox status code (i.e. informational status codes too).
7662 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7663 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7664 *
7665 * @param pVCpu The cross context virtual CPU structure.
7666 * @param u64IntInfo The VM-entry interruption-information field.
7667 * @param cbInstr The VM-entry instruction length in bytes (for
7668 * software interrupts, exceptions and privileged
7669 * software exceptions).
7670 * @param u32ErrCode The VM-entry exception error code.
7671 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7672 * @param pfIntrState Pointer to the current guest interruptibility-state.
7673 * This interruptibility-state will be updated if
7674 * necessary. This cannot not be NULL.
7675 * @param fStepping Whether we're running in
7676 * hmR0VmxRunGuestCodeStep() and should return
7677 * VINF_EM_DBG_STEPPED if the event is injected
7678 * directly (register modified by us, not by
7679 * hardware on VM-entry).
7680 */
7681static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7682 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7683{
7684 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7685 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7686 Assert(pfIntrState);
7687
7688 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7689 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7690 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7691 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7692
7693#ifdef VBOX_STRICT
7694 /*
7695 * Validate the error-code-valid bit for hardware exceptions.
7696 * No error codes for exceptions in real-mode.
7697 *
7698 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7699 */
7700 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7701 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7702 {
7703 switch (uVector)
7704 {
7705 case X86_XCPT_PF:
7706 case X86_XCPT_DF:
7707 case X86_XCPT_TS:
7708 case X86_XCPT_NP:
7709 case X86_XCPT_SS:
7710 case X86_XCPT_GP:
7711 case X86_XCPT_AC:
7712 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7713 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7714 RT_FALL_THRU();
7715 default:
7716 break;
7717 }
7718 }
7719#endif
7720
7721 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7722 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7723 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7724
7725 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7726
7727 /*
7728 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7729 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7730 * interrupt handler in the (real-mode) guest.
7731 *
7732 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7733 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7734 */
7735 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7736 {
7737 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7738 {
7739 /*
7740 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7741 * set the deliver-error-code bit.
7742 *
7743 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7744 */
7745 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7746 }
7747 else
7748 {
7749 PVM pVM = pVCpu->CTX_SUFF(pVM);
7750 Assert(PDMVmmDevHeapIsEnabled(pVM));
7751 Assert(pVM->hm.s.vmx.pRealModeTSS);
7752
7753 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7754 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7755 | CPUMCTX_EXTRN_TABLE_MASK
7756 | CPUMCTX_EXTRN_RIP
7757 | CPUMCTX_EXTRN_RSP
7758 | CPUMCTX_EXTRN_RFLAGS);
7759 AssertRCReturn(rc2, rc2);
7760
7761 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7762 size_t const cbIdtEntry = sizeof(X86IDTR16);
7763 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7764 {
7765 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7766 if (uVector == X86_XCPT_DF)
7767 return VINF_EM_RESET;
7768
7769 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7770 if (uVector == X86_XCPT_GP)
7771 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7772
7773 /*
7774 * If we're injecting an event with no valid IDT entry, inject a #GP.
7775 * No error codes for exceptions in real-mode.
7776 *
7777 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7778 */
7779 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7780 pfIntrState);
7781 }
7782
7783 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7784 uint16_t uGuestIp = pMixedCtx->ip;
7785 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7786 {
7787 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7788 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7789 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7790 }
7791 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7792 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7793
7794 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7795 X86IDTR16 IdtEntry;
7796 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7797 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7798 AssertRCReturn(rc2, rc2);
7799
7800 /* Construct the stack frame for the interrupt/exception handler. */
7801 VBOXSTRICTRC rcStrict;
7802 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7803 if (rcStrict == VINF_SUCCESS)
7804 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7805 if (rcStrict == VINF_SUCCESS)
7806 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7807
7808 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7809 if (rcStrict == VINF_SUCCESS)
7810 {
7811 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7812 pMixedCtx->rip = IdtEntry.offSel;
7813 pMixedCtx->cs.Sel = IdtEntry.uSel;
7814 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7815 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7816 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7817 && uVector == X86_XCPT_PF)
7818 pMixedCtx->cr2 = GCPtrFaultAddress;
7819
7820 /* If any other guest-state bits are changed here, make sure to update
7821 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7822 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7823 | HM_CHANGED_GUEST_CR2
7824 | HM_CHANGED_GUEST_RIP
7825 | HM_CHANGED_GUEST_RFLAGS
7826 | HM_CHANGED_GUEST_RSP);
7827
7828 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7829 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7830 {
7831 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7832 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7833 Log4Func(("Clearing inhibition due to STI\n"));
7834 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7835 }
7836 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7837 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7838
7839 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7840 it, if we are returning to ring-3 before executing guest code. */
7841 pVCpu->hm.s.Event.fPending = false;
7842
7843 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7844 if (fStepping)
7845 rcStrict = VINF_EM_DBG_STEPPED;
7846 }
7847 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7848 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7849 return rcStrict;
7850 }
7851 }
7852
7853 /* Validate. */
7854 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7855 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7856 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7857
7858 /* Inject. */
7859 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7860 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7861 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7862 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7863 AssertRCReturn(rc, rc);
7864
7865 /* Update CR2. */
7866 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7867 && uVector == X86_XCPT_PF)
7868 pMixedCtx->cr2 = GCPtrFaultAddress;
7869
7870 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7871
7872 return VINF_SUCCESS;
7873}
7874
7875
7876/**
7877 * Clears the interrupt-window exiting control in the VMCS and if necessary
7878 * clears the current event in the VMCS as well.
7879 *
7880 * @returns VBox status code.
7881 * @param pVCpu The cross context virtual CPU structure.
7882 *
7883 * @remarks Use this function only to clear events that have not yet been
7884 * delivered to the guest but are injected in the VMCS!
7885 * @remarks No-long-jump zone!!!
7886 */
7887static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7888{
7889 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7890 {
7891 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7892 Log4Func(("Cleared interrupt widow\n"));
7893 }
7894
7895 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7896 {
7897 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7898 Log4Func(("Cleared interrupt widow\n"));
7899 }
7900}
7901
7902
7903/**
7904 * Enters the VT-x session.
7905 *
7906 * @returns VBox status code.
7907 * @param pVCpu The cross context virtual CPU structure.
7908 * @param pHostCpu Pointer to the global CPU info struct.
7909 */
7910VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7911{
7912 AssertPtr(pVCpu);
7913 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7914 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7915 RT_NOREF(pHostCpu);
7916
7917 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7918 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7919 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7920
7921#ifdef VBOX_STRICT
7922 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7923 RTCCUINTREG uHostCR4 = ASMGetCR4();
7924 if (!(uHostCR4 & X86_CR4_VMXE))
7925 {
7926 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7927 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7928 }
7929#endif
7930
7931 /*
7932 * Load the VCPU's VMCS as the current (and active) one.
7933 */
7934 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7935 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7936 if (RT_FAILURE(rc))
7937 return rc;
7938
7939 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7940 pVCpu->hm.s.fLeaveDone = false;
7941 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7942
7943 return VINF_SUCCESS;
7944}
7945
7946
7947/**
7948 * The thread-context callback (only on platforms which support it).
7949 *
7950 * @param enmEvent The thread-context event.
7951 * @param pVCpu The cross context virtual CPU structure.
7952 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7953 * @thread EMT(pVCpu)
7954 */
7955VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7956{
7957 NOREF(fGlobalInit);
7958
7959 switch (enmEvent)
7960 {
7961 case RTTHREADCTXEVENT_OUT:
7962 {
7963 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7964 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7965 VMCPU_ASSERT_EMT(pVCpu);
7966
7967 /* No longjmps (logger flushes, locks) in this fragile context. */
7968 VMMRZCallRing3Disable(pVCpu);
7969 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7970
7971 /*
7972 * Restore host-state (FPU, debug etc.)
7973 */
7974 if (!pVCpu->hm.s.fLeaveDone)
7975 {
7976 /*
7977 * Do -not- import the guest-state here as we might already be in the middle of importing
7978 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7979 */
7980 hmR0VmxLeave(pVCpu, false /* fImportState */);
7981 pVCpu->hm.s.fLeaveDone = true;
7982 }
7983
7984 /* Leave HM context, takes care of local init (term). */
7985 int rc = HMR0LeaveCpu(pVCpu);
7986 AssertRC(rc); NOREF(rc);
7987
7988 /* Restore longjmp state. */
7989 VMMRZCallRing3Enable(pVCpu);
7990 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7991 break;
7992 }
7993
7994 case RTTHREADCTXEVENT_IN:
7995 {
7996 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7997 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7998 VMCPU_ASSERT_EMT(pVCpu);
7999
8000 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
8001 VMMRZCallRing3Disable(pVCpu);
8002 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
8003
8004 /* Initialize the bare minimum state required for HM. This takes care of
8005 initializing VT-x if necessary (onlined CPUs, local init etc.) */
8006 int rc = hmR0EnterCpu(pVCpu);
8007 AssertRC(rc);
8008 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8009 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
8010
8011 /* Load the active VMCS as the current one. */
8012 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
8013 {
8014 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
8015 AssertRC(rc); NOREF(rc);
8016 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
8017 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
8018 }
8019 pVCpu->hm.s.fLeaveDone = false;
8020
8021 /* Restore longjmp state. */
8022 VMMRZCallRing3Enable(pVCpu);
8023 break;
8024 }
8025
8026 default:
8027 break;
8028 }
8029}
8030
8031
8032/**
8033 * Exports the host state into the VMCS host-state area.
8034 * Sets up the VM-exit MSR-load area.
8035 *
8036 * The CPU state will be loaded from these fields on every successful VM-exit.
8037 *
8038 * @returns VBox status code.
8039 * @param pVCpu The cross context virtual CPU structure.
8040 *
8041 * @remarks No-long-jump zone!!!
8042 */
8043static int hmR0VmxExportHostState(PVMCPU pVCpu)
8044{
8045 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8046
8047 int rc = VINF_SUCCESS;
8048 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8049 {
8050 rc = hmR0VmxExportHostControlRegs();
8051 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8052
8053 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8054 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8055
8056 rc = hmR0VmxExportHostMsrs(pVCpu);
8057 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8058
8059 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8060 }
8061 return rc;
8062}
8063
8064
8065/**
8066 * Saves the host state in the VMCS host-state.
8067 *
8068 * @returns VBox status code.
8069 * @param pVCpu The cross context virtual CPU structure.
8070 *
8071 * @remarks No-long-jump zone!!!
8072 */
8073VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8074{
8075 AssertPtr(pVCpu);
8076 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8077
8078 /*
8079 * Export the host state here while entering HM context.
8080 * When thread-context hooks are used, we might get preempted and have to re-save the host
8081 * state but most of the time we won't be, so do it here before we disable interrupts.
8082 */
8083 return hmR0VmxExportHostState(pVCpu);
8084}
8085
8086
8087/**
8088 * Exports the guest state into the VMCS guest-state area.
8089 *
8090 * The will typically be done before VM-entry when the guest-CPU state and the
8091 * VMCS state may potentially be out of sync.
8092 *
8093 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8094 * VM-entry controls.
8095 * Sets up the appropriate VMX non-root function to execute guest code based on
8096 * the guest CPU mode.
8097 *
8098 * @returns VBox strict status code.
8099 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8100 * without unrestricted guest access and the VMMDev is not presently
8101 * mapped (e.g. EFI32).
8102 *
8103 * @param pVCpu The cross context virtual CPU structure.
8104 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8105 * out-of-sync. Make sure to update the required fields
8106 * before using them.
8107 *
8108 * @remarks No-long-jump zone!!!
8109 */
8110static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8111{
8112 AssertPtr(pVCpu);
8113 AssertPtr(pMixedCtx);
8114 HMVMX_ASSERT_PREEMPT_SAFE();
8115
8116 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8117
8118 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8119
8120 /* Determine real-on-v86 mode. */
8121 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8122 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8123 && CPUMIsGuestInRealModeEx(pMixedCtx))
8124 {
8125 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8126 }
8127
8128 /*
8129 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8130 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8131 */
8132 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8133 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8134
8135 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8136 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8137 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8138
8139 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8140 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8141 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8142
8143 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8144 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8145
8146 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8147 if (rcStrict == VINF_SUCCESS)
8148 { /* likely */ }
8149 else
8150 {
8151 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8152 return rcStrict;
8153 }
8154
8155 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8156 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8157
8158 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8159 may alter controls if we determine we don't have to swap EFER after all. */
8160 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8161 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8162
8163 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8164 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8165
8166 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8167 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8168
8169 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8170 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8171 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8172 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8173 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8174 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8175
8176 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8177 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8178 | HM_CHANGED_GUEST_CR2
8179 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8180 | HM_CHANGED_GUEST_X87
8181 | HM_CHANGED_GUEST_SSE_AVX
8182 | HM_CHANGED_GUEST_OTHER_XSAVE
8183 | HM_CHANGED_GUEST_XCRx
8184 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8185 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8186 | HM_CHANGED_GUEST_TSC_AUX
8187 | HM_CHANGED_GUEST_OTHER_MSRS
8188 | HM_CHANGED_GUEST_HWVIRT
8189 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8190
8191 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8192 return rc;
8193}
8194
8195
8196/**
8197 * Exports the state shared between the host and guest into the VMCS.
8198 *
8199 * @param pVCpu The cross context virtual CPU structure.
8200 * @param pCtx Pointer to the guest-CPU context.
8201 *
8202 * @remarks No-long-jump zone!!!
8203 */
8204static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8205{
8206 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8207 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8208
8209 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8210 {
8211 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8212 AssertRC(rc);
8213 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8214
8215 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8216 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8217 {
8218 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8219 AssertRC(rc);
8220 }
8221 }
8222
8223 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8224 {
8225 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8226 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8227 }
8228
8229 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8230 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8231}
8232
8233
8234/**
8235 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8236 *
8237 * @returns Strict VBox status code (i.e. informational status codes too).
8238 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8239 * without unrestricted guest access and the VMMDev is not presently
8240 * mapped (e.g. EFI32).
8241 *
8242 * @param pVCpu The cross context virtual CPU structure.
8243 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8244 * out-of-sync. Make sure to update the required fields
8245 * before using them.
8246 *
8247 * @remarks No-long-jump zone!!!
8248 */
8249static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8250{
8251 HMVMX_ASSERT_PREEMPT_SAFE();
8252 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8253 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8254
8255#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8256 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8257#endif
8258
8259 /*
8260 * For many exits it's only RIP that changes and hence try to export it first
8261 * without going through a lot of change flag checks.
8262 */
8263 VBOXSTRICTRC rcStrict;
8264 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8265 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8266 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8267 {
8268 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8269 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8270 { /* likely */}
8271 else
8272 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8273 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8274 }
8275 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8276 {
8277 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8278 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8279 { /* likely */}
8280 else
8281 {
8282 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8283 VBOXSTRICTRC_VAL(rcStrict)));
8284 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8285 return rcStrict;
8286 }
8287 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8288 }
8289 else
8290 rcStrict = VINF_SUCCESS;
8291
8292#ifdef VBOX_STRICT
8293 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8294 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8295 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8296 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8297 ("fCtxChanged=%#RX64\n", fCtxChanged));
8298#endif
8299 return rcStrict;
8300}
8301
8302
8303/**
8304 * Does the preparations before executing guest code in VT-x.
8305 *
8306 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8307 * recompiler/IEM. We must be cautious what we do here regarding committing
8308 * guest-state information into the VMCS assuming we assuredly execute the
8309 * guest in VT-x mode.
8310 *
8311 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8312 * the common-state (TRPM/forceflags), we must undo those changes so that the
8313 * recompiler/IEM can (and should) use them when it resumes guest execution.
8314 * Otherwise such operations must be done when we can no longer exit to ring-3.
8315 *
8316 * @returns Strict VBox status code (i.e. informational status codes too).
8317 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8318 * have been disabled.
8319 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8320 * double-fault into the guest.
8321 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8322 * dispatched directly.
8323 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8324 *
8325 * @param pVCpu The cross context virtual CPU structure.
8326 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8327 * out-of-sync. Make sure to update the required fields
8328 * before using them.
8329 * @param pVmxTransient Pointer to the VMX transient structure.
8330 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8331 * us ignore some of the reasons for returning to
8332 * ring-3, and return VINF_EM_DBG_STEPPED if event
8333 * dispatching took place.
8334 */
8335static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8336{
8337 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8338
8339#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8340 PGMRZDynMapFlushAutoSet(pVCpu);
8341#endif
8342
8343 /* Check force flag actions that might require us to go back to ring-3. */
8344 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8345 if (rcStrict == VINF_SUCCESS)
8346 { /* FFs doesn't get set all the time. */ }
8347 else
8348 return rcStrict;
8349
8350 /*
8351 * Setup the virtualized-APIC accesses.
8352 *
8353 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8354 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8355 *
8356 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8357 */
8358 PVM pVM = pVCpu->CTX_SUFF(pVM);
8359 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8360 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8361 && PDMHasApic(pVM))
8362 {
8363 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8364 Assert(u64MsrApicBase);
8365 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8366
8367 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8368
8369 /* Unalias any existing mapping. */
8370 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8371 AssertRCReturn(rc, rc);
8372
8373 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8374 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8375 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8376 AssertRCReturn(rc, rc);
8377
8378 /* Update the per-VCPU cache of the APIC base MSR. */
8379 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8380 }
8381
8382 if (TRPMHasTrap(pVCpu))
8383 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8384 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8385
8386 /*
8387 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8388 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8389 * also result in triple-faulting the VM.
8390 */
8391 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8392 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8393 { /* likely */ }
8394 else
8395 {
8396 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8397 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8398 return rcStrict;
8399 }
8400
8401 /*
8402 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8403 * import CR3 themselves. We will need to update them here as even as late as the above
8404 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8405 * the below force flags to be set.
8406 */
8407 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8408 {
8409 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8410 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8411 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8412 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8413 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8414 }
8415 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8416 {
8417 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8418 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8419 }
8420
8421 /*
8422 * No longjmps to ring-3 from this point on!!!
8423 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8424 * This also disables flushing of the R0-logger instance (if any).
8425 */
8426 VMMRZCallRing3Disable(pVCpu);
8427
8428 /*
8429 * Export the guest state bits.
8430 *
8431 * We cannot perform longjmps while loading the guest state because we do not preserve the
8432 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8433 * CPU migration.
8434 *
8435 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8436 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8437 * Hence, loading of the guest state needs to be done -after- injection of events.
8438 */
8439 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8440 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8441 { /* likely */ }
8442 else
8443 {
8444 VMMRZCallRing3Enable(pVCpu);
8445 return rcStrict;
8446 }
8447
8448 /*
8449 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8450 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8451 * preemption disabled for a while. Since this is purly to aid the
8452 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8453 * disable interrupt on NT.
8454 *
8455 * We need to check for force-flags that could've possible been altered since we last
8456 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8457 * see @bugref{6398}).
8458 *
8459 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8460 * to ring-3 before executing guest code.
8461 */
8462 pVmxTransient->fEFlags = ASMIntDisableFlags();
8463
8464 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8465 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8466 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8467 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8468 {
8469 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8470 {
8471 pVCpu->hm.s.Event.fPending = false;
8472
8473 /*
8474 * We've injected any pending events. This is really the point of no return (to ring-3).
8475 *
8476 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8477 * returns from this function, so don't enable them here.
8478 */
8479 return VINF_SUCCESS;
8480 }
8481
8482 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8483 rcStrict = VINF_EM_RAW_INTERRUPT;
8484 }
8485 else
8486 {
8487 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8488 rcStrict = VINF_EM_RAW_TO_R3;
8489 }
8490
8491 ASMSetFlags(pVmxTransient->fEFlags);
8492 VMMRZCallRing3Enable(pVCpu);
8493
8494 return rcStrict;
8495}
8496
8497
8498/**
8499 * Prepares to run guest code in VT-x and we've committed to doing so. This
8500 * means there is no backing out to ring-3 or anywhere else at this
8501 * point.
8502 *
8503 * @param pVCpu The cross context virtual CPU structure.
8504 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8505 * out-of-sync. Make sure to update the required fields
8506 * before using them.
8507 * @param pVmxTransient Pointer to the VMX transient structure.
8508 *
8509 * @remarks Called with preemption disabled.
8510 * @remarks No-long-jump zone!!!
8511 */
8512static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8513{
8514 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8515 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8516 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8517
8518 /*
8519 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8520 */
8521 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8522 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8523
8524 PVM pVM = pVCpu->CTX_SUFF(pVM);
8525 if (!CPUMIsGuestFPUStateActive(pVCpu))
8526 {
8527 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8528 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8529 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8530 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8531 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8532 }
8533
8534 /*
8535 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8536 */
8537 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8538 && pVCpu->hm.s.vmx.cMsrs > 0)
8539 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8540
8541 /*
8542 * Re-save the host state bits as we may've been preempted (only happens when
8543 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8544 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8545 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8546 * See @bugref{8432}.
8547 */
8548 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8549 {
8550 int rc = hmR0VmxExportHostState(pVCpu);
8551 AssertRC(rc);
8552 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8553 }
8554 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8555
8556 /*
8557 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8558 */
8559 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8560 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8561 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8562
8563 /* Store status of the shared guest-host state at the time of VM-entry. */
8564#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8565 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8566 {
8567 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8568 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8569 }
8570 else
8571#endif
8572 {
8573 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8574 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8575 }
8576
8577 /*
8578 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8579 */
8580 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8581 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8582
8583 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8584 RTCPUID idCurrentCpu = pCpu->idCpu;
8585 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8586 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8587 {
8588 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8589 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8590 }
8591
8592 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8593 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8594 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8595 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8596
8597 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8598
8599 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8600 to start executing. */
8601
8602 /*
8603 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8604 */
8605 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8606 {
8607 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8608 {
8609 bool fMsrUpdated;
8610 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8611 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8612 &fMsrUpdated);
8613 AssertRC(rc2);
8614 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8615 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8616 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8617 }
8618 else
8619 {
8620 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8621 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8622 }
8623 }
8624
8625 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8626 {
8627 bool fMsrUpdated;
8628 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8629 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8630 &fMsrUpdated);
8631 AssertRC(rc2);
8632 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8633 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8634 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8635 }
8636
8637#ifdef VBOX_STRICT
8638 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8639 hmR0VmxCheckHostEferMsr(pVCpu);
8640 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8641#endif
8642#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8643 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8644 {
8645 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8646 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8647 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8648 }
8649#endif
8650}
8651
8652
8653/**
8654 * Performs some essential restoration of state after running guest code in
8655 * VT-x.
8656 *
8657 * @param pVCpu The cross context virtual CPU structure.
8658 * @param pVmxTransient Pointer to the VMX transient structure.
8659 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8660 *
8661 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8662 *
8663 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8664 * unconditionally when it is safe to do so.
8665 */
8666static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8667{
8668 uint64_t const uHostTsc = ASMReadTSC();
8669 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8670
8671 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8672 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8673 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8674 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8675 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8676 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8677
8678 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8679 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8680
8681 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8682 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8683 Assert(!ASMIntAreEnabled());
8684 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8685
8686#if HC_ARCH_BITS == 64
8687 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8688#endif
8689#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8690 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8691 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8692 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8693#else
8694 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8695#endif
8696#ifdef VBOX_STRICT
8697 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8698#endif
8699 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8700
8701 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8702 uint32_t uExitReason;
8703 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8704 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8705 AssertRC(rc);
8706 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8707 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8708
8709 if (rcVMRun == VINF_SUCCESS)
8710 {
8711 /*
8712 * Update the VM-exit history array here even if the VM-entry failed due to:
8713 * - Invalid guest state.
8714 * - MSR loading.
8715 * - Machine-check event.
8716 *
8717 * In any of the above cases we will still have a "valid" VM-exit reason
8718 * despite @a fVMEntryFailed being false.
8719 *
8720 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8721 *
8722 * Note! We don't have CS or RIP at this point. Will probably address that later
8723 * by amending the history entry added here.
8724 */
8725 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8726 UINT64_MAX, uHostTsc);
8727
8728 if (!pVmxTransient->fVMEntryFailed)
8729 {
8730 VMMRZCallRing3Enable(pVCpu);
8731
8732 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8733 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8734
8735#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8736 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8737 AssertRC(rc);
8738#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8739 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8740 AssertRC(rc);
8741#else
8742 /*
8743 * Import the guest-interruptibility state always as we need it while evaluating
8744 * injecting events on re-entry.
8745 *
8746 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8747 * checking for real-mode while exporting the state because all bits that cause
8748 * mode changes wrt CR0 are intercepted.
8749 */
8750 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8751 AssertRC(rc);
8752#endif
8753
8754 /*
8755 * Sync the TPR shadow with our APIC state.
8756 */
8757 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8758 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8759 {
8760 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8761 AssertRC(rc);
8762 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8763 }
8764
8765 return;
8766 }
8767 }
8768 else
8769 {
8770 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8771 }
8772
8773 VMMRZCallRing3Enable(pVCpu);
8774}
8775
8776
8777/**
8778 * Runs the guest code using VT-x the normal way.
8779 *
8780 * @returns VBox status code.
8781 * @param pVCpu The cross context virtual CPU structure.
8782 * @param pCtx Pointer to the guest-CPU context.
8783 *
8784 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8785 */
8786static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8787{
8788 VMXTRANSIENT VmxTransient;
8789 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8790 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8791 uint32_t cLoops = 0;
8792
8793 for (;; cLoops++)
8794 {
8795 Assert(!HMR0SuspendPending());
8796 HMVMX_ASSERT_CPU_SAFE();
8797
8798 /* Preparatory work for running guest code, this may force us to return
8799 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8800 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8801 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8802 if (rcStrict != VINF_SUCCESS)
8803 break;
8804
8805 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8806 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8807 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8808
8809 /* Restore any residual host-state and save any bits shared between host
8810 and guest into the guest-CPU state. Re-enables interrupts! */
8811 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8812
8813 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8814 if (RT_SUCCESS(rcRun))
8815 { /* very likely */ }
8816 else
8817 {
8818 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8819 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8820 return rcRun;
8821 }
8822
8823 /* Profile the VM-exit. */
8824 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8826 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8827 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8828 HMVMX_START_EXIT_DISPATCH_PROF();
8829
8830 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8831
8832 /* Handle the VM-exit. */
8833#ifdef HMVMX_USE_FUNCTION_TABLE
8834 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8835#else
8836 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8837#endif
8838 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8839 if (rcStrict == VINF_SUCCESS)
8840 {
8841 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8842 continue; /* likely */
8843 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8844 rcStrict = VINF_EM_RAW_INTERRUPT;
8845 }
8846 break;
8847 }
8848
8849 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8850 return rcStrict;
8851}
8852
8853
8854
8855/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8856 * probes.
8857 *
8858 * The following few functions and associated structure contains the bloat
8859 * necessary for providing detailed debug events and dtrace probes as well as
8860 * reliable host side single stepping. This works on the principle of
8861 * "subclassing" the normal execution loop and workers. We replace the loop
8862 * method completely and override selected helpers to add necessary adjustments
8863 * to their core operation.
8864 *
8865 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8866 * any performance for debug and analysis features.
8867 *
8868 * @{
8869 */
8870
8871/**
8872 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8873 * the debug run loop.
8874 */
8875typedef struct VMXRUNDBGSTATE
8876{
8877 /** The RIP we started executing at. This is for detecting that we stepped. */
8878 uint64_t uRipStart;
8879 /** The CS we started executing with. */
8880 uint16_t uCsStart;
8881
8882 /** Whether we've actually modified the 1st execution control field. */
8883 bool fModifiedProcCtls : 1;
8884 /** Whether we've actually modified the 2nd execution control field. */
8885 bool fModifiedProcCtls2 : 1;
8886 /** Whether we've actually modified the exception bitmap. */
8887 bool fModifiedXcptBitmap : 1;
8888
8889 /** We desire the modified the CR0 mask to be cleared. */
8890 bool fClearCr0Mask : 1;
8891 /** We desire the modified the CR4 mask to be cleared. */
8892 bool fClearCr4Mask : 1;
8893 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8894 uint32_t fCpe1Extra;
8895 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8896 uint32_t fCpe1Unwanted;
8897 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8898 uint32_t fCpe2Extra;
8899 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8900 uint32_t bmXcptExtra;
8901 /** The sequence number of the Dtrace provider settings the state was
8902 * configured against. */
8903 uint32_t uDtraceSettingsSeqNo;
8904 /** VM-exits to check (one bit per VM-exit). */
8905 uint32_t bmExitsToCheck[3];
8906
8907 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8908 uint32_t fProcCtlsInitial;
8909 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8910 uint32_t fProcCtls2Initial;
8911 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8912 uint32_t bmXcptInitial;
8913} VMXRUNDBGSTATE;
8914AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8915typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8916
8917
8918/**
8919 * Initializes the VMXRUNDBGSTATE structure.
8920 *
8921 * @param pVCpu The cross context virtual CPU structure of the
8922 * calling EMT.
8923 * @param pCtx The CPU register context to go with @a pVCpu.
8924 * @param pDbgState The structure to initialize.
8925 */
8926static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8927{
8928 pDbgState->uRipStart = pCtx->rip;
8929 pDbgState->uCsStart = pCtx->cs.Sel;
8930
8931 pDbgState->fModifiedProcCtls = false;
8932 pDbgState->fModifiedProcCtls2 = false;
8933 pDbgState->fModifiedXcptBitmap = false;
8934 pDbgState->fClearCr0Mask = false;
8935 pDbgState->fClearCr4Mask = false;
8936 pDbgState->fCpe1Extra = 0;
8937 pDbgState->fCpe1Unwanted = 0;
8938 pDbgState->fCpe2Extra = 0;
8939 pDbgState->bmXcptExtra = 0;
8940 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8941 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8942 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8943}
8944
8945
8946/**
8947 * Updates the VMSC fields with changes requested by @a pDbgState.
8948 *
8949 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8950 * immediately before executing guest code, i.e. when interrupts are disabled.
8951 * We don't check status codes here as we cannot easily assert or return in the
8952 * latter case.
8953 *
8954 * @param pVCpu The cross context virtual CPU structure.
8955 * @param pDbgState The debug state.
8956 */
8957static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8958{
8959 /*
8960 * Ensure desired flags in VMCS control fields are set.
8961 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8962 *
8963 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8964 * there should be no stale data in pCtx at this point.
8965 */
8966 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8967 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8968 {
8969 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8970 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8971 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8972 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8973 pDbgState->fModifiedProcCtls = true;
8974 }
8975
8976 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8977 {
8978 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8979 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8980 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8981 pDbgState->fModifiedProcCtls2 = true;
8982 }
8983
8984 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8985 {
8986 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8987 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8988 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8989 pDbgState->fModifiedXcptBitmap = true;
8990 }
8991
8992 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8993 {
8994 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8995 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8996 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8997 }
8998
8999 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
9000 {
9001 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
9002 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
9003 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
9004 }
9005}
9006
9007
9008static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
9009{
9010 /*
9011 * Restore VM-exit control settings as we may not reenter this function the
9012 * next time around.
9013 */
9014 /* We reload the initial value, trigger what we can of recalculations the
9015 next time around. From the looks of things, that's all that's required atm. */
9016 if (pDbgState->fModifiedProcCtls)
9017 {
9018 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
9019 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
9020 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
9021 AssertRCReturn(rc2, rc2);
9022 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
9023 }
9024
9025 /* We're currently the only ones messing with this one, so just restore the
9026 cached value and reload the field. */
9027 if ( pDbgState->fModifiedProcCtls2
9028 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9029 {
9030 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9031 AssertRCReturn(rc2, rc2);
9032 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9033 }
9034
9035 /* If we've modified the exception bitmap, we restore it and trigger
9036 reloading and partial recalculation the next time around. */
9037 if (pDbgState->fModifiedXcptBitmap)
9038 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9039
9040 return rcStrict;
9041}
9042
9043
9044/**
9045 * Configures VM-exit controls for current DBGF and DTrace settings.
9046 *
9047 * This updates @a pDbgState and the VMCS execution control fields to reflect
9048 * the necessary VM-exits demanded by DBGF and DTrace.
9049 *
9050 * @param pVCpu The cross context virtual CPU structure.
9051 * @param pDbgState The debug state.
9052 * @param pVmxTransient Pointer to the VMX transient structure. May update
9053 * fUpdateTscOffsettingAndPreemptTimer.
9054 */
9055static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9056{
9057 /*
9058 * Take down the dtrace serial number so we can spot changes.
9059 */
9060 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9061 ASMCompilerBarrier();
9062
9063 /*
9064 * We'll rebuild most of the middle block of data members (holding the
9065 * current settings) as we go along here, so start by clearing it all.
9066 */
9067 pDbgState->bmXcptExtra = 0;
9068 pDbgState->fCpe1Extra = 0;
9069 pDbgState->fCpe1Unwanted = 0;
9070 pDbgState->fCpe2Extra = 0;
9071 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9072 pDbgState->bmExitsToCheck[i] = 0;
9073
9074 /*
9075 * Software interrupts (INT XXh) - no idea how to trigger these...
9076 */
9077 PVM pVM = pVCpu->CTX_SUFF(pVM);
9078 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9079 || VBOXVMM_INT_SOFTWARE_ENABLED())
9080 {
9081 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9082 }
9083
9084 /*
9085 * INT3 breakpoints - triggered by #BP exceptions.
9086 */
9087 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9088 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9089
9090 /*
9091 * Exception bitmap and XCPT events+probes.
9092 */
9093 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9094 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9095 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9096
9097 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9098 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9099 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9100 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9101 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9102 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9103 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9104 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9105 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9106 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9107 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9108 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9109 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9110 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9111 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9112 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9113 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9114 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9115
9116 if (pDbgState->bmXcptExtra)
9117 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9118
9119 /*
9120 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9121 *
9122 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9123 * So, when adding/changing/removing please don't forget to update it.
9124 *
9125 * Some of the macros are picking up local variables to save horizontal space,
9126 * (being able to see it in a table is the lesser evil here).
9127 */
9128#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9129 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9130 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9131#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9132 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9133 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9134 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9135 } else do { } while (0)
9136#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9137 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9138 { \
9139 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9140 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9141 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9142 } else do { } while (0)
9143#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9144 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9145 { \
9146 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9147 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9148 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9149 } else do { } while (0)
9150#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9151 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9152 { \
9153 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9154 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9155 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9156 } else do { } while (0)
9157
9158 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9159 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9160 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9161 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9162 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9163
9164 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9166 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9168 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9170 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9172 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9174 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9176 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9177 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9178 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9180 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9181 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9182 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9184 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9185 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9186 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9187 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9188 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9189 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9190 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9191 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9192 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9193 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9194 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9195 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9196 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9197 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9198 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9199 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9200
9201 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9202 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9203 {
9204 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9205 | CPUMCTX_EXTRN_CR4
9206 | CPUMCTX_EXTRN_APIC_TPR);
9207 AssertRC(rc);
9208
9209#if 0 /** @todo fix me */
9210 pDbgState->fClearCr0Mask = true;
9211 pDbgState->fClearCr4Mask = true;
9212#endif
9213 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9214 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9215 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9216 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9217 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9218 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9219 require clearing here and in the loop if we start using it. */
9220 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9221 }
9222 else
9223 {
9224 if (pDbgState->fClearCr0Mask)
9225 {
9226 pDbgState->fClearCr0Mask = false;
9227 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9228 }
9229 if (pDbgState->fClearCr4Mask)
9230 {
9231 pDbgState->fClearCr4Mask = false;
9232 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9233 }
9234 }
9235 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9236 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9237
9238 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9239 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9240 {
9241 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9242 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9243 }
9244 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9245 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9246
9247 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9248 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9249 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9250 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9251 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9252 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9253 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9254 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9255#if 0 /** @todo too slow, fix handler. */
9256 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9257#endif
9258 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9259
9260 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9261 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9262 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9263 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9264 {
9265 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9266 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9267 }
9268 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9269 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9270 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9271 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9272
9273 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9274 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9275 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9276 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9277 {
9278 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9279 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9280 }
9281 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9282 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9284 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9285
9286 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9287 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9288 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9289 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9290 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9291 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9292 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9293 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9294 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9295 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9296 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9297 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9298 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9299 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9300 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9301 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9302 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9303 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9304 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9305 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9306 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9307 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9308
9309#undef IS_EITHER_ENABLED
9310#undef SET_ONLY_XBM_IF_EITHER_EN
9311#undef SET_CPE1_XBM_IF_EITHER_EN
9312#undef SET_CPEU_XBM_IF_EITHER_EN
9313#undef SET_CPE2_XBM_IF_EITHER_EN
9314
9315 /*
9316 * Sanitize the control stuff.
9317 */
9318 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9319 if (pDbgState->fCpe2Extra)
9320 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9321 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9322 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9323 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9324 {
9325 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9326 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9327 }
9328
9329 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9330 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9331 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9332 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9333}
9334
9335
9336/**
9337 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9338 * appropriate.
9339 *
9340 * The caller has checked the VM-exit against the
9341 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9342 * already, so we don't have to do that either.
9343 *
9344 * @returns Strict VBox status code (i.e. informational status codes too).
9345 * @param pVCpu The cross context virtual CPU structure.
9346 * @param pMixedCtx Pointer to the guest-CPU context.
9347 * @param pVmxTransient Pointer to the VMX-transient structure.
9348 * @param uExitReason The VM-exit reason.
9349 *
9350 * @remarks The name of this function is displayed by dtrace, so keep it short
9351 * and to the point. No longer than 33 chars long, please.
9352 */
9353static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9354 uint32_t uExitReason)
9355{
9356 /*
9357 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9358 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9359 *
9360 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9361 * does. Must add/change/remove both places. Same ordering, please.
9362 *
9363 * Added/removed events must also be reflected in the next section
9364 * where we dispatch dtrace events.
9365 */
9366 bool fDtrace1 = false;
9367 bool fDtrace2 = false;
9368 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9369 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9370 uint32_t uEventArg = 0;
9371#define SET_EXIT(a_EventSubName) \
9372 do { \
9373 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9374 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9375 } while (0)
9376#define SET_BOTH(a_EventSubName) \
9377 do { \
9378 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9379 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9380 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9381 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9382 } while (0)
9383 switch (uExitReason)
9384 {
9385 case VMX_EXIT_MTF:
9386 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9387
9388 case VMX_EXIT_XCPT_OR_NMI:
9389 {
9390 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9391 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9392 {
9393 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9394 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9395 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9396 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9397 {
9398 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9399 {
9400 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9401 uEventArg = pVmxTransient->uExitIntErrorCode;
9402 }
9403 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9404 switch (enmEvent1)
9405 {
9406 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9407 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9408 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9409 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9410 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9411 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9412 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9413 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9414 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9415 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9416 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9417 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9418 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9419 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9420 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9421 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9422 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9423 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9424 default: break;
9425 }
9426 }
9427 else
9428 AssertFailed();
9429 break;
9430
9431 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9432 uEventArg = idxVector;
9433 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9434 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9435 break;
9436 }
9437 break;
9438 }
9439
9440 case VMX_EXIT_TRIPLE_FAULT:
9441 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9442 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9443 break;
9444 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9445 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9446 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9447 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9448 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9449
9450 /* Instruction specific VM-exits: */
9451 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9452 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9453 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9454 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9455 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9456 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9457 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9458 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9459 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9460 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9461 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9462 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9463 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9464 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9465 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9466 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9467 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9468 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9469 case VMX_EXIT_MOV_CRX:
9470 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9471 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9472 SET_BOTH(CRX_READ);
9473 else
9474 SET_BOTH(CRX_WRITE);
9475 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9476 break;
9477 case VMX_EXIT_MOV_DRX:
9478 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9479 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9480 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9481 SET_BOTH(DRX_READ);
9482 else
9483 SET_BOTH(DRX_WRITE);
9484 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9485 break;
9486 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9487 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9488 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9489 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9490 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9491 case VMX_EXIT_XDTR_ACCESS:
9492 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9493 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9494 {
9495 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9496 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9497 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9498 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9499 }
9500 break;
9501
9502 case VMX_EXIT_TR_ACCESS:
9503 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9504 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9505 {
9506 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9507 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9508 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9509 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9510 }
9511 break;
9512
9513 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9514 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9515 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9516 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9517 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9518 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9519 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9520 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9521 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9522 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9523 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9524
9525 /* Events that aren't relevant at this point. */
9526 case VMX_EXIT_EXT_INT:
9527 case VMX_EXIT_INT_WINDOW:
9528 case VMX_EXIT_NMI_WINDOW:
9529 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9530 case VMX_EXIT_PREEMPT_TIMER:
9531 case VMX_EXIT_IO_INSTR:
9532 break;
9533
9534 /* Errors and unexpected events. */
9535 case VMX_EXIT_INIT_SIGNAL:
9536 case VMX_EXIT_SIPI:
9537 case VMX_EXIT_IO_SMI:
9538 case VMX_EXIT_SMI:
9539 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9540 case VMX_EXIT_ERR_MSR_LOAD:
9541 case VMX_EXIT_ERR_MACHINE_CHECK:
9542 break;
9543
9544 default:
9545 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9546 break;
9547 }
9548#undef SET_BOTH
9549#undef SET_EXIT
9550
9551 /*
9552 * Dtrace tracepoints go first. We do them here at once so we don't
9553 * have to copy the guest state saving and stuff a few dozen times.
9554 * Down side is that we've got to repeat the switch, though this time
9555 * we use enmEvent since the probes are a subset of what DBGF does.
9556 */
9557 if (fDtrace1 || fDtrace2)
9558 {
9559 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9560 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9561 switch (enmEvent1)
9562 {
9563 /** @todo consider which extra parameters would be helpful for each probe. */
9564 case DBGFEVENT_END: break;
9565 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9567 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9574 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9575 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9576 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9577 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9578 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9583 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9584 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9585 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9593 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9594 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9595 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9596 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9597 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9598 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9599 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9607 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9608 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9609 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9610 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9615 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9617 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9623 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9625 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9626 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9627 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9628 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9631 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9632 }
9633 switch (enmEvent2)
9634 {
9635 /** @todo consider which extra parameters would be helpful for each probe. */
9636 case DBGFEVENT_END: break;
9637 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9638 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9639 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9647 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9648 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9649 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9650 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9651 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9652 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9653 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9665 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9666 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9667 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9668 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9669 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9670 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9671 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9672 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9673 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9674 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9675 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9676 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9677 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9678 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9679 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9680 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9681 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9682 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9683 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9684 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9685 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9686 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9687 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9688 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9689 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9690 }
9691 }
9692
9693 /*
9694 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9695 * the DBGF call will do a full check).
9696 *
9697 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9698 * Note! If we have to events, we prioritize the first, i.e. the instruction
9699 * one, in order to avoid event nesting.
9700 */
9701 PVM pVM = pVCpu->CTX_SUFF(pVM);
9702 if ( enmEvent1 != DBGFEVENT_END
9703 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9704 {
9705 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9706 if (rcStrict != VINF_SUCCESS)
9707 return rcStrict;
9708 }
9709 else if ( enmEvent2 != DBGFEVENT_END
9710 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9711 {
9712 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9713 if (rcStrict != VINF_SUCCESS)
9714 return rcStrict;
9715 }
9716
9717 return VINF_SUCCESS;
9718}
9719
9720
9721/**
9722 * Single-stepping VM-exit filtering.
9723 *
9724 * This is preprocessing the VM-exits and deciding whether we've gotten far
9725 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9726 * handling is performed.
9727 *
9728 * @returns Strict VBox status code (i.e. informational status codes too).
9729 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9730 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9731 * out-of-sync. Make sure to update the required
9732 * fields before using them.
9733 * @param pVmxTransient Pointer to the VMX-transient structure.
9734 * @param uExitReason The VM-exit reason.
9735 * @param pDbgState The debug state.
9736 */
9737DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9738 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9739{
9740 /*
9741 * Expensive (saves context) generic dtrace VM-exit probe.
9742 */
9743 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9744 { /* more likely */ }
9745 else
9746 {
9747 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9748 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9749 AssertRC(rc);
9750 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9751 }
9752
9753 /*
9754 * Check for host NMI, just to get that out of the way.
9755 */
9756 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9757 { /* normally likely */ }
9758 else
9759 {
9760 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9761 AssertRCReturn(rc2, rc2);
9762 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9763 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9764 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9765 }
9766
9767 /*
9768 * Check for single stepping event if we're stepping.
9769 */
9770 if (pVCpu->hm.s.fSingleInstruction)
9771 {
9772 switch (uExitReason)
9773 {
9774 case VMX_EXIT_MTF:
9775 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9776
9777 /* Various events: */
9778 case VMX_EXIT_XCPT_OR_NMI:
9779 case VMX_EXIT_EXT_INT:
9780 case VMX_EXIT_TRIPLE_FAULT:
9781 case VMX_EXIT_INT_WINDOW:
9782 case VMX_EXIT_NMI_WINDOW:
9783 case VMX_EXIT_TASK_SWITCH:
9784 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9785 case VMX_EXIT_APIC_ACCESS:
9786 case VMX_EXIT_EPT_VIOLATION:
9787 case VMX_EXIT_EPT_MISCONFIG:
9788 case VMX_EXIT_PREEMPT_TIMER:
9789
9790 /* Instruction specific VM-exits: */
9791 case VMX_EXIT_CPUID:
9792 case VMX_EXIT_GETSEC:
9793 case VMX_EXIT_HLT:
9794 case VMX_EXIT_INVD:
9795 case VMX_EXIT_INVLPG:
9796 case VMX_EXIT_RDPMC:
9797 case VMX_EXIT_RDTSC:
9798 case VMX_EXIT_RSM:
9799 case VMX_EXIT_VMCALL:
9800 case VMX_EXIT_VMCLEAR:
9801 case VMX_EXIT_VMLAUNCH:
9802 case VMX_EXIT_VMPTRLD:
9803 case VMX_EXIT_VMPTRST:
9804 case VMX_EXIT_VMREAD:
9805 case VMX_EXIT_VMRESUME:
9806 case VMX_EXIT_VMWRITE:
9807 case VMX_EXIT_VMXOFF:
9808 case VMX_EXIT_VMXON:
9809 case VMX_EXIT_MOV_CRX:
9810 case VMX_EXIT_MOV_DRX:
9811 case VMX_EXIT_IO_INSTR:
9812 case VMX_EXIT_RDMSR:
9813 case VMX_EXIT_WRMSR:
9814 case VMX_EXIT_MWAIT:
9815 case VMX_EXIT_MONITOR:
9816 case VMX_EXIT_PAUSE:
9817 case VMX_EXIT_XDTR_ACCESS:
9818 case VMX_EXIT_TR_ACCESS:
9819 case VMX_EXIT_INVEPT:
9820 case VMX_EXIT_RDTSCP:
9821 case VMX_EXIT_INVVPID:
9822 case VMX_EXIT_WBINVD:
9823 case VMX_EXIT_XSETBV:
9824 case VMX_EXIT_RDRAND:
9825 case VMX_EXIT_INVPCID:
9826 case VMX_EXIT_VMFUNC:
9827 case VMX_EXIT_RDSEED:
9828 case VMX_EXIT_XSAVES:
9829 case VMX_EXIT_XRSTORS:
9830 {
9831 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9832 | CPUMCTX_EXTRN_CS);
9833 AssertRCReturn(rc, rc);
9834 if ( pMixedCtx->rip != pDbgState->uRipStart
9835 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9836 return VINF_EM_DBG_STEPPED;
9837 break;
9838 }
9839
9840 /* Errors and unexpected events: */
9841 case VMX_EXIT_INIT_SIGNAL:
9842 case VMX_EXIT_SIPI:
9843 case VMX_EXIT_IO_SMI:
9844 case VMX_EXIT_SMI:
9845 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9846 case VMX_EXIT_ERR_MSR_LOAD:
9847 case VMX_EXIT_ERR_MACHINE_CHECK:
9848 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9849 break;
9850
9851 default:
9852 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9853 break;
9854 }
9855 }
9856
9857 /*
9858 * Check for debugger event breakpoints and dtrace probes.
9859 */
9860 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9861 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9862 {
9863 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9864 if (rcStrict != VINF_SUCCESS)
9865 return rcStrict;
9866 }
9867
9868 /*
9869 * Normal processing.
9870 */
9871#ifdef HMVMX_USE_FUNCTION_TABLE
9872 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9873#else
9874 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9875#endif
9876}
9877
9878
9879/**
9880 * Single steps guest code using VT-x.
9881 *
9882 * @returns Strict VBox status code (i.e. informational status codes too).
9883 * @param pVCpu The cross context virtual CPU structure.
9884 * @param pCtx Pointer to the guest-CPU context.
9885 *
9886 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9887 */
9888static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9889{
9890 VMXTRANSIENT VmxTransient;
9891 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9892
9893 /* Set HMCPU indicators. */
9894 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9895 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9896 pVCpu->hm.s.fDebugWantRdTscExit = false;
9897 pVCpu->hm.s.fUsingDebugLoop = true;
9898
9899 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9900 VMXRUNDBGSTATE DbgState;
9901 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9902 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9903
9904 /*
9905 * The loop.
9906 */
9907 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9908 for (uint32_t cLoops = 0; ; cLoops++)
9909 {
9910 Assert(!HMR0SuspendPending());
9911 HMVMX_ASSERT_CPU_SAFE();
9912 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9913
9914 /*
9915 * Preparatory work for running guest code, this may force us to return
9916 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9917 */
9918 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9919 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9920 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9921 if (rcStrict != VINF_SUCCESS)
9922 break;
9923
9924 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9925 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9926
9927 /*
9928 * Now we can run the guest code.
9929 */
9930 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9931
9932 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9933
9934 /*
9935 * Restore any residual host-state and save any bits shared between host
9936 * and guest into the guest-CPU state. Re-enables interrupts!
9937 */
9938 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9939
9940 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9941 if (RT_SUCCESS(rcRun))
9942 { /* very likely */ }
9943 else
9944 {
9945 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9946 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9947 return rcRun;
9948 }
9949
9950 /* Profile the VM-exit. */
9951 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9952 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9953 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9954 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9955 HMVMX_START_EXIT_DISPATCH_PROF();
9956
9957 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9958
9959 /*
9960 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9961 */
9962 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9963 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9964 if (rcStrict != VINF_SUCCESS)
9965 break;
9966 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9967 {
9968 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9969 rcStrict = VINF_EM_RAW_INTERRUPT;
9970 break;
9971 }
9972
9973 /*
9974 * Stepping: Did the RIP change, if so, consider it a single step.
9975 * Otherwise, make sure one of the TFs gets set.
9976 */
9977 if (fStepping)
9978 {
9979 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9980 | CPUMCTX_EXTRN_CS);
9981 AssertRC(rc);
9982 if ( pCtx->rip != DbgState.uRipStart
9983 || pCtx->cs.Sel != DbgState.uCsStart)
9984 {
9985 rcStrict = VINF_EM_DBG_STEPPED;
9986 break;
9987 }
9988 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9989 }
9990
9991 /*
9992 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9993 */
9994 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9995 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9996 }
9997
9998 /*
9999 * Clear the X86_EFL_TF if necessary.
10000 */
10001 if (pVCpu->hm.s.fClearTrapFlag)
10002 {
10003 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
10004 AssertRC(rc);
10005 pVCpu->hm.s.fClearTrapFlag = false;
10006 pCtx->eflags.Bits.u1TF = 0;
10007 }
10008 /** @todo there seems to be issues with the resume flag when the monitor trap
10009 * flag is pending without being used. Seen early in bios init when
10010 * accessing APIC page in protected mode. */
10011
10012 /*
10013 * Restore VM-exit control settings as we may not reenter this function the
10014 * next time around.
10015 */
10016 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
10017
10018 /* Restore HMCPU indicators. */
10019 pVCpu->hm.s.fUsingDebugLoop = false;
10020 pVCpu->hm.s.fDebugWantRdTscExit = false;
10021 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
10022
10023 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10024 return rcStrict;
10025}
10026
10027
10028/** @} */
10029
10030
10031/**
10032 * Checks if any expensive dtrace probes are enabled and we should go to the
10033 * debug loop.
10034 *
10035 * @returns true if we should use debug loop, false if not.
10036 */
10037static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10038{
10039 /* It's probably faster to OR the raw 32-bit counter variables together.
10040 Since the variables are in an array and the probes are next to one
10041 another (more or less), we have good locality. So, better read
10042 eight-nine cache lines ever time and only have one conditional, than
10043 128+ conditionals, right? */
10044 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10045 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10046 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10047 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10048 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10049 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10050 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10051 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10052 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10053 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10054 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10055 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10056 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10057 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10058 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10059 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10060 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10061 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10062 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10063 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10064 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10065 ) != 0
10066 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10067 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10068 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10069 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10070 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10071 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10072 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10073 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10074 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10075 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10076 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10077 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10078 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10079 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10080 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10081 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10082 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10083 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10084 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10085 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10086 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10087 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10088 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10089 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10090 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10091 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10092 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10093 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10094 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10095 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10096 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10097 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10098 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10099 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10100 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10101 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10102 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10103 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10104 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10105 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10106 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10107 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10108 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10109 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10110 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10111 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10112 ) != 0
10113 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10114 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10115 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10116 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10117 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10118 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10119 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10120 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10121 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10122 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10123 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10124 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10125 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10126 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10127 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10128 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10129 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10130 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10131 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10132 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10133 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10134 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10135 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10136 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10137 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10138 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10139 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10140 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10141 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10142 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10143 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10144 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10145 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10146 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10147 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10148 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10149 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10150 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10151 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10152 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10153 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10154 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10155 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10156 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10157 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10158 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10159 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10160 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10161 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10162 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10163 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10164 ) != 0;
10165}
10166
10167
10168/**
10169 * Runs the guest code using VT-x.
10170 *
10171 * @returns Strict VBox status code (i.e. informational status codes too).
10172 * @param pVCpu The cross context virtual CPU structure.
10173 * @param pCtx Pointer to the guest-CPU context.
10174 */
10175VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10176{
10177 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10178 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10179 HMVMX_ASSERT_PREEMPT_SAFE();
10180
10181 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10182
10183 VBOXSTRICTRC rcStrict;
10184 if ( !pVCpu->hm.s.fUseDebugLoop
10185 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10186 && !DBGFIsStepping(pVCpu)
10187 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10188 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10189 else
10190 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10191
10192 if (rcStrict == VERR_EM_INTERPRETER)
10193 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10194 else if (rcStrict == VINF_EM_RESET)
10195 rcStrict = VINF_EM_TRIPLE_FAULT;
10196
10197 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10198 if (RT_FAILURE(rc2))
10199 {
10200 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10201 rcStrict = rc2;
10202 }
10203 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10204 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10205 return rcStrict;
10206}
10207
10208
10209#ifndef HMVMX_USE_FUNCTION_TABLE
10210DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10211{
10212#ifdef DEBUG_ramshankar
10213#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10214 do { \
10215 if (a_fSave != 0) \
10216 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10217 VBOXSTRICTRC rcStrict = a_CallExpr; \
10218 if (a_fSave != 0) \
10219 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10220 return rcStrict; \
10221 } while (0)
10222#else
10223# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10224#endif
10225 switch (rcReason)
10226 {
10227 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10228 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10229 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10236 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10237 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10238 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10239 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10240 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10241 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10242 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10243 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10244 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10245 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10246 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10247 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10248 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10249 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10250 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10251 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10252 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10253 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10254 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10255 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10256 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10257 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10258 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10259 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10260 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10261
10262 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10263 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10264 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10265 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10266 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10267 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10268 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10269 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10270 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10271
10272 case VMX_EXIT_VMCLEAR:
10273 case VMX_EXIT_VMLAUNCH:
10274 case VMX_EXIT_VMPTRLD:
10275 case VMX_EXIT_VMPTRST:
10276 case VMX_EXIT_VMREAD:
10277 case VMX_EXIT_VMRESUME:
10278 case VMX_EXIT_VMWRITE:
10279 case VMX_EXIT_VMXOFF:
10280 case VMX_EXIT_VMXON:
10281 case VMX_EXIT_INVEPT:
10282 case VMX_EXIT_INVVPID:
10283 case VMX_EXIT_VMFUNC:
10284 case VMX_EXIT_XSAVES:
10285 case VMX_EXIT_XRSTORS:
10286 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10287
10288 case VMX_EXIT_ENCLS:
10289 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10290 case VMX_EXIT_PML_FULL:
10291 default:
10292 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10293 }
10294#undef VMEXIT_CALL_RET
10295}
10296#endif /* !HMVMX_USE_FUNCTION_TABLE */
10297
10298
10299#ifdef VBOX_STRICT
10300/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10301# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10302 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10303
10304# define HMVMX_ASSERT_PREEMPT_CPUID() \
10305 do { \
10306 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10307 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10308 } while (0)
10309
10310# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10311 do { \
10312 AssertPtr(pVCpu); \
10313 AssertPtr(pMixedCtx); \
10314 AssertPtr(pVmxTransient); \
10315 Assert(pVmxTransient->fVMEntryFailed == false); \
10316 Assert(ASMIntAreEnabled()); \
10317 HMVMX_ASSERT_PREEMPT_SAFE(); \
10318 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10319 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10320 HMVMX_ASSERT_PREEMPT_SAFE(); \
10321 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10322 HMVMX_ASSERT_PREEMPT_CPUID(); \
10323 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10324 } while (0)
10325
10326# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10327 do { \
10328 Log4Func(("\n")); \
10329 } while (0)
10330#else /* nonstrict builds: */
10331# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10332 do { \
10333 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10334 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10335 } while (0)
10336# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10337#endif
10338
10339
10340/**
10341 * Advances the guest RIP by the specified number of bytes.
10342 *
10343 * @param pVCpu The cross context virtual CPU structure.
10344 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10345 * out-of-sync. Make sure to update the required fields
10346 * before using them.
10347 * @param cbInstr Number of bytes to advance the RIP by.
10348 *
10349 * @remarks No-long-jump zone!!!
10350 */
10351DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10352{
10353 /* Advance the RIP. */
10354 pMixedCtx->rip += cbInstr;
10355 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10356
10357 /* Update interrupt inhibition. */
10358 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10359 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10360 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10361}
10362
10363
10364/**
10365 * Advances the guest RIP after reading it from the VMCS.
10366 *
10367 * @returns VBox status code, no informational status codes.
10368 * @param pVCpu The cross context virtual CPU structure.
10369 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10370 * out-of-sync. Make sure to update the required fields
10371 * before using them.
10372 * @param pVmxTransient Pointer to the VMX transient structure.
10373 *
10374 * @remarks No-long-jump zone!!!
10375 */
10376static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10377{
10378 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10379 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10380 | CPUMCTX_EXTRN_RFLAGS);
10381 AssertRCReturn(rc, rc);
10382
10383 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10384
10385 /*
10386 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10387 * pending debug exception field as it takes care of priority of events.
10388 *
10389 * See Intel spec. 32.2.1 "Debug Exceptions".
10390 */
10391 if ( !pVCpu->hm.s.fSingleInstruction
10392 && pMixedCtx->eflags.Bits.u1TF)
10393 {
10394 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10395 AssertRCReturn(rc, rc);
10396 }
10397
10398 return VINF_SUCCESS;
10399}
10400
10401
10402/**
10403 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10404 * and update error record fields accordingly.
10405 *
10406 * @return VMX_IGS_* return codes.
10407 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10408 * wrong with the guest state.
10409 *
10410 * @param pVCpu The cross context virtual CPU structure.
10411 * @param pCtx Pointer to the guest-CPU state.
10412 *
10413 * @remarks This function assumes our cache of the VMCS controls
10414 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10415 */
10416static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10417{
10418#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10419#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10420 uError = (err); \
10421 break; \
10422 } else do { } while (0)
10423
10424 int rc;
10425 PVM pVM = pVCpu->CTX_SUFF(pVM);
10426 uint32_t uError = VMX_IGS_ERROR;
10427 uint32_t u32Val;
10428 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10429
10430 do
10431 {
10432 /*
10433 * CR0.
10434 */
10435 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10436 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10437 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10438 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10439 if (fUnrestrictedGuest)
10440 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10441
10442 uint32_t u32GuestCr0;
10443 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10444 AssertRCBreak(rc);
10445 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10446 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10447 if ( !fUnrestrictedGuest
10448 && (u32GuestCr0 & X86_CR0_PG)
10449 && !(u32GuestCr0 & X86_CR0_PE))
10450 {
10451 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10452 }
10453
10454 /*
10455 * CR4.
10456 */
10457 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10458 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10459
10460 uint32_t u32GuestCr4;
10461 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10462 AssertRCBreak(rc);
10463 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10464 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10465
10466 /*
10467 * IA32_DEBUGCTL MSR.
10468 */
10469 uint64_t u64Val;
10470 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10471 AssertRCBreak(rc);
10472 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10473 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10474 {
10475 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10476 }
10477 uint64_t u64DebugCtlMsr = u64Val;
10478
10479#ifdef VBOX_STRICT
10480 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10481 AssertRCBreak(rc);
10482 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10483#endif
10484 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10485
10486 /*
10487 * RIP and RFLAGS.
10488 */
10489 uint32_t u32Eflags;
10490#if HC_ARCH_BITS == 64
10491 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10492 AssertRCBreak(rc);
10493 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10494 if ( !fLongModeGuest
10495 || !pCtx->cs.Attr.n.u1Long)
10496 {
10497 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10498 }
10499 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10500 * must be identical if the "IA-32e mode guest" VM-entry
10501 * control is 1 and CS.L is 1. No check applies if the
10502 * CPU supports 64 linear-address bits. */
10503
10504 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10505 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10506 AssertRCBreak(rc);
10507 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10508 VMX_IGS_RFLAGS_RESERVED);
10509 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10510 u32Eflags = u64Val;
10511#else
10512 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10513 AssertRCBreak(rc);
10514 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10515 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10516#endif
10517
10518 if ( fLongModeGuest
10519 || ( fUnrestrictedGuest
10520 && !(u32GuestCr0 & X86_CR0_PE)))
10521 {
10522 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10523 }
10524
10525 uint32_t u32EntryInfo;
10526 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10527 AssertRCBreak(rc);
10528 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10529 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10530 {
10531 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10532 }
10533
10534 /*
10535 * 64-bit checks.
10536 */
10537#if HC_ARCH_BITS == 64
10538 if (fLongModeGuest)
10539 {
10540 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10541 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10542 }
10543
10544 if ( !fLongModeGuest
10545 && (u32GuestCr4 & X86_CR4_PCIDE))
10546 {
10547 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10548 }
10549
10550 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10551 * 51:32 beyond the processor's physical-address width are 0. */
10552
10553 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10554 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10555 {
10556 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10557 }
10558
10559 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10560 AssertRCBreak(rc);
10561 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10562
10563 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10564 AssertRCBreak(rc);
10565 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10566#endif
10567
10568 /*
10569 * PERF_GLOBAL MSR.
10570 */
10571 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10572 {
10573 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10574 AssertRCBreak(rc);
10575 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10576 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10577 }
10578
10579 /*
10580 * PAT MSR.
10581 */
10582 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10583 {
10584 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10585 AssertRCBreak(rc);
10586 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10587 for (unsigned i = 0; i < 8; i++)
10588 {
10589 uint8_t u8Val = (u64Val & 0xff);
10590 if ( u8Val != 0 /* UC */
10591 && u8Val != 1 /* WC */
10592 && u8Val != 4 /* WT */
10593 && u8Val != 5 /* WP */
10594 && u8Val != 6 /* WB */
10595 && u8Val != 7 /* UC- */)
10596 {
10597 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10598 }
10599 u64Val >>= 8;
10600 }
10601 }
10602
10603 /*
10604 * EFER MSR.
10605 */
10606 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10607 {
10608 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10609 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10610 AssertRCBreak(rc);
10611 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10612 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10613 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10614 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10615 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10616 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10617 || !(u32GuestCr0 & X86_CR0_PG)
10618 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10619 VMX_IGS_EFER_LMA_LME_MISMATCH);
10620 }
10621
10622 /*
10623 * Segment registers.
10624 */
10625 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10626 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10627 if (!(u32Eflags & X86_EFL_VM))
10628 {
10629 /* CS */
10630 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10631 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10632 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10633 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10634 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10635 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10636 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10637 /* CS cannot be loaded with NULL in protected mode. */
10638 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10639 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10640 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10641 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10642 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10643 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10644 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10645 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10646 else
10647 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10648
10649 /* SS */
10650 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10651 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10652 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10653 if ( !(pCtx->cr0 & X86_CR0_PE)
10654 || pCtx->cs.Attr.n.u4Type == 3)
10655 {
10656 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10657 }
10658 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10659 {
10660 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10661 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10662 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10663 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10664 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10665 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10666 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10667 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10668 }
10669
10670 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10671 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10672 {
10673 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10674 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10675 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10676 || pCtx->ds.Attr.n.u4Type > 11
10677 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10678 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10679 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10680 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10681 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10682 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10683 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10684 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10685 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10686 }
10687 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10688 {
10689 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10690 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10691 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10692 || pCtx->es.Attr.n.u4Type > 11
10693 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10694 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10695 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10696 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10697 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10698 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10699 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10700 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10701 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10702 }
10703 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10704 {
10705 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10706 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10707 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10708 || pCtx->fs.Attr.n.u4Type > 11
10709 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10710 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10711 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10712 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10713 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10714 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10715 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10716 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10717 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10718 }
10719 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10720 {
10721 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10722 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10723 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10724 || pCtx->gs.Attr.n.u4Type > 11
10725 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10726 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10727 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10728 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10729 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10730 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10731 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10732 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10733 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10734 }
10735 /* 64-bit capable CPUs. */
10736#if HC_ARCH_BITS == 64
10737 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10738 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10739 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10740 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10741 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10742 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10743 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10744 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10745 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10746 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10747 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10748#endif
10749 }
10750 else
10751 {
10752 /* V86 mode checks. */
10753 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10754 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10755 {
10756 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10757 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10758 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10759 }
10760 else
10761 {
10762 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10763 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10764 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10765 }
10766
10767 /* CS */
10768 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10769 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10770 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10771 /* SS */
10772 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10773 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10774 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10775 /* DS */
10776 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10777 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10778 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10779 /* ES */
10780 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10781 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10782 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10783 /* FS */
10784 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10785 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10786 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10787 /* GS */
10788 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10789 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10790 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10791 /* 64-bit capable CPUs. */
10792#if HC_ARCH_BITS == 64
10793 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10794 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10795 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10796 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10797 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10798 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10799 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10800 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10801 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10802 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10803 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10804#endif
10805 }
10806
10807 /*
10808 * TR.
10809 */
10810 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10811 /* 64-bit capable CPUs. */
10812#if HC_ARCH_BITS == 64
10813 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10814#endif
10815 if (fLongModeGuest)
10816 {
10817 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10818 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10819 }
10820 else
10821 {
10822 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10823 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10824 VMX_IGS_TR_ATTR_TYPE_INVALID);
10825 }
10826 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10827 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10828 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10829 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10830 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10831 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10832 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10833 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10834
10835 /*
10836 * GDTR and IDTR.
10837 */
10838#if HC_ARCH_BITS == 64
10839 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10840 AssertRCBreak(rc);
10841 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10842
10843 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10844 AssertRCBreak(rc);
10845 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10846#endif
10847
10848 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10849 AssertRCBreak(rc);
10850 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10851
10852 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10853 AssertRCBreak(rc);
10854 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10855
10856 /*
10857 * Guest Non-Register State.
10858 */
10859 /* Activity State. */
10860 uint32_t u32ActivityState;
10861 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10862 AssertRCBreak(rc);
10863 HMVMX_CHECK_BREAK( !u32ActivityState
10864 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10865 VMX_IGS_ACTIVITY_STATE_INVALID);
10866 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10867 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10868 uint32_t u32IntrState;
10869 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10870 AssertRCBreak(rc);
10871 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10872 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10873 {
10874 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10875 }
10876
10877 /** @todo Activity state and injecting interrupts. Left as a todo since we
10878 * currently don't use activity states but ACTIVE. */
10879
10880 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10881 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10882
10883 /* Guest interruptibility-state. */
10884 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10885 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10886 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10887 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10888 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10889 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10890 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10891 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10892 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10893 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10894 {
10895 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10896 {
10897 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10898 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10899 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10900 }
10901 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10902 {
10903 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10904 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10905 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10906 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10907 }
10908 }
10909 /** @todo Assumes the processor is not in SMM. */
10910 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10911 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10912 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10913 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10914 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10915 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10916 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10917 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10918 {
10919 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10920 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10921 }
10922
10923 /* Pending debug exceptions. */
10924#if HC_ARCH_BITS == 64
10925 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10926 AssertRCBreak(rc);
10927 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10928 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10929 u32Val = u64Val; /* For pending debug exceptions checks below. */
10930#else
10931 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10932 AssertRCBreak(rc);
10933 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10934 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10935#endif
10936
10937 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10938 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10939 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10940 {
10941 if ( (u32Eflags & X86_EFL_TF)
10942 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10943 {
10944 /* Bit 14 is PendingDebug.BS. */
10945 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10946 }
10947 if ( !(u32Eflags & X86_EFL_TF)
10948 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10949 {
10950 /* Bit 14 is PendingDebug.BS. */
10951 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10952 }
10953 }
10954
10955 /* VMCS link pointer. */
10956 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10957 AssertRCBreak(rc);
10958 if (u64Val != UINT64_C(0xffffffffffffffff))
10959 {
10960 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10961 /** @todo Bits beyond the processor's physical-address width MBZ. */
10962 /** @todo 32-bit located in memory referenced by value of this field (as a
10963 * physical address) must contain the processor's VMCS revision ID. */
10964 /** @todo SMM checks. */
10965 }
10966
10967 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10968 * not using Nested Paging? */
10969 if ( pVM->hm.s.fNestedPaging
10970 && !fLongModeGuest
10971 && CPUMIsGuestInPAEModeEx(pCtx))
10972 {
10973 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10974 AssertRCBreak(rc);
10975 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10976
10977 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10978 AssertRCBreak(rc);
10979 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10980
10981 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10982 AssertRCBreak(rc);
10983 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10984
10985 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10986 AssertRCBreak(rc);
10987 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10988 }
10989
10990 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10991 if (uError == VMX_IGS_ERROR)
10992 uError = VMX_IGS_REASON_NOT_FOUND;
10993 } while (0);
10994
10995 pVCpu->hm.s.u32HMError = uError;
10996 return uError;
10997
10998#undef HMVMX_ERROR_BREAK
10999#undef HMVMX_CHECK_BREAK
11000}
11001
11002/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11003/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
11004/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
11005
11006/** @name VM-exit handlers.
11007 * @{
11008 */
11009
11010/**
11011 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
11012 */
11013HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11014{
11015 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
11017 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
11018 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
11019 return VINF_SUCCESS;
11020 return VINF_EM_RAW_INTERRUPT;
11021}
11022
11023
11024/**
11025 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11026 */
11027HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11028{
11029 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11030 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11031
11032 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11033 AssertRCReturn(rc, rc);
11034
11035 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11036 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11037 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11038 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11039
11040 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11041 {
11042 /*
11043 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11044 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11045 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11046 *
11047 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11048 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11049 */
11050 VMXDispatchHostNmi();
11051 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11052 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11053 return VINF_SUCCESS;
11054 }
11055
11056 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11057 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11058 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11059 { /* likely */ }
11060 else
11061 {
11062 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11063 rcStrictRc1 = VINF_SUCCESS;
11064 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11065 return rcStrictRc1;
11066 }
11067
11068 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11069 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11070 switch (uIntType)
11071 {
11072 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11073 Assert(uVector == X86_XCPT_DB);
11074 RT_FALL_THRU();
11075 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11076 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11077 RT_FALL_THRU();
11078 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11079 {
11080 /*
11081 * If there's any exception caused as a result of event injection, the resulting
11082 * secondary/final execption will be pending, we shall continue guest execution
11083 * after injecting the event. The page-fault case is complicated and we manually
11084 * handle any currently pending event in hmR0VmxExitXcptPF.
11085 */
11086 if (!pVCpu->hm.s.Event.fPending)
11087 { /* likely */ }
11088 else if (uVector != X86_XCPT_PF)
11089 {
11090 rc = VINF_SUCCESS;
11091 break;
11092 }
11093
11094 switch (uVector)
11095 {
11096 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11097 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11098 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11099 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11100 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11101 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11102
11103 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11104 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11105 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11106 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11107 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11108 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11109 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11110 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11111 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11112 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11113 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11114 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11115 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11116 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11117 default:
11118 {
11119 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11120 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11121 {
11122 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11123 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11124 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11125
11126 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11127 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11128 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11129 AssertRCReturn(rc, rc);
11130 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11131 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11132 0 /* GCPtrFaultAddress */);
11133 }
11134 else
11135 {
11136 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11137 pVCpu->hm.s.u32HMError = uVector;
11138 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11139 }
11140 break;
11141 }
11142 }
11143 break;
11144 }
11145
11146 default:
11147 {
11148 pVCpu->hm.s.u32HMError = uExitIntInfo;
11149 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11150 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11151 break;
11152 }
11153 }
11154 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11155 return rc;
11156}
11157
11158
11159/**
11160 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11161 */
11162HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11163{
11164 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11165
11166 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11167 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11168
11169 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11170 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11171 return VINF_SUCCESS;
11172}
11173
11174
11175/**
11176 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11177 */
11178HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11179{
11180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11181 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11182 {
11183 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11184 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11185 }
11186
11187 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11188
11189 /*
11190 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11191 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11192 */
11193 uint32_t fIntrState = 0;
11194 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11195 AssertRCReturn(rc, rc);
11196
11197 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11198 if ( fBlockSti
11199 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11200 {
11201 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11202 }
11203
11204 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11205 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11206
11207 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11208 return VINF_SUCCESS;
11209}
11210
11211
11212/**
11213 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11214 */
11215HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11216{
11217 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11218 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11219}
11220
11221
11222/**
11223 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11224 */
11225HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11226{
11227 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11228 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11229}
11230
11231
11232/**
11233 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11234 */
11235HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11236{
11237 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11238 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11239
11240 /*
11241 * Get the state we need and update the exit history entry.
11242 */
11243 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11244 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11245 | CPUMCTX_EXTRN_CS);
11246 AssertRCReturn(rc, rc);
11247
11248 VBOXSTRICTRC rcStrict;
11249 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11250 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11251 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11252 if (!pExitRec)
11253 {
11254 /*
11255 * Regular CPUID instruction execution.
11256 */
11257 PVM pVM = pVCpu->CTX_SUFF(pVM);
11258 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11259 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11260 {
11261 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11262 Assert(pVmxTransient->cbInstr == 2);
11263 }
11264 else
11265 {
11266 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11267 rcStrict = VERR_EM_INTERPRETER;
11268 }
11269 }
11270 else
11271 {
11272 /*
11273 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11274 */
11275 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11276 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11277 AssertRCReturn(rc2, rc2);
11278
11279 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11280 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11281
11282 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11283 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11284
11285 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11286 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11287 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11288 }
11289 return VBOXSTRICTRC_TODO(rcStrict);
11290}
11291
11292
11293/**
11294 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11295 */
11296HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11297{
11298 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11299 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11300 AssertRCReturn(rc, rc);
11301
11302 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11303 return VINF_EM_RAW_EMULATE_INSTR;
11304
11305 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11306 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11307}
11308
11309
11310/**
11311 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11312 */
11313HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11314{
11315 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11316 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11317 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11318 AssertRCReturn(rc, rc);
11319
11320 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11321 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11322 {
11323 /* If we get a spurious VM-exit when offsetting is enabled,
11324 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11325 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11326 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11327 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11328 | HM_CHANGED_GUEST_RFLAGS);
11329 }
11330 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11331 {
11332 rcStrict = VINF_SUCCESS;
11333 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11334 }
11335 return rcStrict;
11336}
11337
11338
11339/**
11340 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11341 */
11342HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11343{
11344 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11345 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11346 | CPUMCTX_EXTRN_TSC_AUX);
11347 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11348 AssertRCReturn(rc, rc);
11349
11350 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11351 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11352 {
11353 /* If we get a spurious VM-exit when offsetting is enabled,
11354 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11355 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11356 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11357 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11358 | HM_CHANGED_GUEST_RFLAGS);
11359 }
11360 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11361 {
11362 rcStrict = VINF_SUCCESS;
11363 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11364 }
11365 return rcStrict;
11366}
11367
11368
11369/**
11370 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11371 */
11372HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11373{
11374 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11375 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11376 | CPUMCTX_EXTRN_CR0
11377 | CPUMCTX_EXTRN_RFLAGS
11378 | CPUMCTX_EXTRN_SS);
11379 AssertRCReturn(rc, rc);
11380
11381 PVM pVM = pVCpu->CTX_SUFF(pVM);
11382 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11383 if (RT_LIKELY(rc == VINF_SUCCESS))
11384 {
11385 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11386 Assert(pVmxTransient->cbInstr == 2);
11387 }
11388 else
11389 {
11390 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11391 rc = VERR_EM_INTERPRETER;
11392 }
11393 return rc;
11394}
11395
11396
11397/**
11398 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11399 */
11400HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11401{
11402 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11403
11404 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11405 if (EMAreHypercallInstructionsEnabled(pVCpu))
11406 {
11407 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11408 | CPUMCTX_EXTRN_RFLAGS
11409 | CPUMCTX_EXTRN_CR0
11410 | CPUMCTX_EXTRN_SS
11411 | CPUMCTX_EXTRN_CS
11412 | CPUMCTX_EXTRN_EFER);
11413 AssertRCReturn(rc, rc);
11414
11415 /* Perform the hypercall. */
11416 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11417 if (rcStrict == VINF_SUCCESS)
11418 {
11419 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11420 AssertRCReturn(rc, rc);
11421 }
11422 else
11423 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11424 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11425 || RT_FAILURE(rcStrict));
11426
11427 /* If the hypercall changes anything other than guest's general-purpose registers,
11428 we would need to reload the guest changed bits here before VM-entry. */
11429 }
11430 else
11431 Log4Func(("Hypercalls not enabled\n"));
11432
11433 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11434 if (RT_FAILURE(rcStrict))
11435 {
11436 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11437 rcStrict = VINF_SUCCESS;
11438 }
11439
11440 return rcStrict;
11441}
11442
11443
11444/**
11445 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11446 */
11447HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11448{
11449 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11450 PVM pVM = pVCpu->CTX_SUFF(pVM);
11451 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11452
11453 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11454 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11455 AssertRCReturn(rc, rc);
11456
11457 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11458 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11459 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11460 else
11461 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11462 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11463 return rcStrict;
11464}
11465
11466
11467/**
11468 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11469 */
11470HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11471{
11472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11473 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11474 | CPUMCTX_EXTRN_RFLAGS
11475 | CPUMCTX_EXTRN_SS);
11476 AssertRCReturn(rc, rc);
11477
11478 PVM pVM = pVCpu->CTX_SUFF(pVM);
11479 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11480 if (RT_LIKELY(rc == VINF_SUCCESS))
11481 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11482 else
11483 {
11484 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11485 rc = VERR_EM_INTERPRETER;
11486 }
11487 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11488 return rc;
11489}
11490
11491
11492/**
11493 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11494 */
11495HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11496{
11497 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11498 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11499 | CPUMCTX_EXTRN_RFLAGS
11500 | CPUMCTX_EXTRN_SS);
11501 AssertRCReturn(rc, rc);
11502
11503 PVM pVM = pVCpu->CTX_SUFF(pVM);
11504 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11505 rc = VBOXSTRICTRC_VAL(rc2);
11506 if (RT_LIKELY( rc == VINF_SUCCESS
11507 || rc == VINF_EM_HALT))
11508 {
11509 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11510 AssertRCReturn(rc3, rc3);
11511
11512 if ( rc == VINF_EM_HALT
11513 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11514 rc = VINF_SUCCESS;
11515 }
11516 else
11517 {
11518 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11519 rc = VERR_EM_INTERPRETER;
11520 }
11521 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11522 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11523 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11524 return rc;
11525}
11526
11527
11528/**
11529 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11530 */
11531HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11532{
11533 /*
11534 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11535 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11536 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11537 * VMX root operation. If we get here, something funny is going on.
11538 *
11539 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11540 */
11541 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11542 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11543 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11544}
11545
11546
11547/**
11548 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11549 */
11550HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11551{
11552 /*
11553 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11554 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11555 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11556 * an SMI. If we get here, something funny is going on.
11557 *
11558 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11559 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11560 */
11561 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11562 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11563 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11564}
11565
11566
11567/**
11568 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11569 */
11570HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11571{
11572 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11574 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11575 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11576}
11577
11578
11579/**
11580 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11581 */
11582HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11583{
11584 /*
11585 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11586 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11587 * See Intel spec. 25.3 "Other Causes of VM-exits".
11588 */
11589 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11590 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11591 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11592}
11593
11594
11595/**
11596 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11597 * VM-exit.
11598 */
11599HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11600{
11601 /*
11602 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11603 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11604 *
11605 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11606 * See Intel spec. "23.8 Restrictions on VMX operation".
11607 */
11608 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11609 return VINF_SUCCESS;
11610}
11611
11612
11613/**
11614 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11615 * VM-exit.
11616 */
11617HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11618{
11619 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11620 return VINF_EM_RESET;
11621}
11622
11623
11624/**
11625 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11626 */
11627HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11628{
11629 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11630 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11631
11632 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11633 AssertRCReturn(rc, rc);
11634
11635 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11636 rc = VINF_SUCCESS;
11637 else
11638 rc = VINF_EM_HALT;
11639
11640 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11641 if (rc != VINF_SUCCESS)
11642 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11643 return rc;
11644}
11645
11646
11647/**
11648 * VM-exit handler for instructions that result in a \#UD exception delivered to
11649 * the guest.
11650 */
11651HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11652{
11653 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11654 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11655 return VINF_SUCCESS;
11656}
11657
11658
11659/**
11660 * VM-exit handler for expiry of the VMX preemption timer.
11661 */
11662HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11663{
11664 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11665
11666 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11667 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11668
11669 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11670 PVM pVM = pVCpu->CTX_SUFF(pVM);
11671 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11672 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11673 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11674}
11675
11676
11677/**
11678 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11679 */
11680HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11681{
11682 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11683
11684 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11685 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11686 | CPUMCTX_EXTRN_CR4);
11687 AssertRCReturn(rc, rc);
11688
11689 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11690 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11691 : HM_CHANGED_XCPT_RAISED_MASK);
11692
11693 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11694
11695 return rcStrict;
11696}
11697
11698
11699/**
11700 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11701 */
11702HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11703{
11704 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11705 /** @todo Use VM-exit instruction information. */
11706 return VERR_EM_INTERPRETER;
11707}
11708
11709
11710/**
11711 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11712 * Error VM-exit.
11713 */
11714HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11715{
11716 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11717 AssertRCReturn(rc, rc);
11718 rc = hmR0VmxCheckVmcsCtls(pVCpu);
11719 if (RT_FAILURE(rc))
11720 return rc;
11721
11722 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11723 NOREF(uInvalidReason);
11724
11725#ifdef VBOX_STRICT
11726 uint32_t fIntrState;
11727 RTHCUINTREG uHCReg;
11728 uint64_t u64Val;
11729 uint32_t u32Val;
11730
11731 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11732 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11733 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11734 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11735 AssertRCReturn(rc, rc);
11736
11737 Log4(("uInvalidReason %u\n", uInvalidReason));
11738 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11739 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11740 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11741 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11742
11743 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11744 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11745 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11746 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11747 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11748 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11749 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11750 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11751 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11752 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11753 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11754 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11755
11756 hmR0DumpRegs(pVCpu, pMixedCtx);
11757#else
11758 NOREF(pVmxTransient);
11759#endif
11760
11761 return VERR_VMX_INVALID_GUEST_STATE;
11762}
11763
11764
11765/**
11766 * VM-exit handler for VM-entry failure due to an MSR-load
11767 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11768 */
11769HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11770{
11771 NOREF(pVmxTransient);
11772 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11773 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11774}
11775
11776
11777/**
11778 * VM-exit handler for VM-entry failure due to a machine-check event
11779 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11780 */
11781HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11782{
11783 NOREF(pVmxTransient);
11784 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11785 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11786}
11787
11788
11789/**
11790 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11791 * theory.
11792 */
11793HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11794{
11795 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11796 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11797 return VERR_VMX_UNDEFINED_EXIT_CODE;
11798}
11799
11800
11801/**
11802 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11803 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11804 * Conditional VM-exit.
11805 */
11806HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11807{
11808 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11809
11810 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11811 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11812 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11813 return VERR_EM_INTERPRETER;
11814 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11815 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11816}
11817
11818
11819/**
11820 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11821 */
11822HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11823{
11824 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11825
11826 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11827 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11828 return VERR_EM_INTERPRETER;
11829 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11830 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11831}
11832
11833
11834/**
11835 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11836 */
11837HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11838{
11839 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11840
11841 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11842 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11843 | CPUMCTX_EXTRN_RFLAGS
11844 | CPUMCTX_EXTRN_SS
11845 | CPUMCTX_EXTRN_FS
11846 | CPUMCTX_EXTRN_GS);
11847 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11848 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11849 AssertRCReturn(rc, rc);
11850 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11851
11852#ifdef VBOX_STRICT
11853 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11854 {
11855 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11856 && pMixedCtx->ecx != MSR_K6_EFER)
11857 {
11858 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11859 pMixedCtx->ecx));
11860 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11861 }
11862 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11863 {
11864 VMXMSREXITREAD enmRead;
11865 VMXMSREXITWRITE enmWrite;
11866 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11867 AssertRCReturn(rc2, rc2);
11868 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11869 {
11870 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11871 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11872 }
11873 }
11874 }
11875#endif
11876
11877 PVM pVM = pVCpu->CTX_SUFF(pVM);
11878 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11879 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11880 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11881 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11882 if (RT_SUCCESS(rc))
11883 {
11884 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11885 Assert(pVmxTransient->cbInstr == 2);
11886 }
11887 return rc;
11888}
11889
11890
11891/**
11892 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11893 */
11894HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11895{
11896 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11897
11898 /** @todo Optimize this: We currently drag in in the whole MSR state
11899 * (CPUMCTX_EXTRN_ALL_MSRS) here. We should optimize this to only get
11900 * MSRs required. That would require changes to IEM and possibly CPUM too.
11901 * (Should probably do it lazy fashion from CPUMAllMsrs.cpp). */
11902 uint32_t const idMsr = pMixedCtx->ecx; /* Save it. */
11903 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11904 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_EXEC_DECODED_NO_MEM_MASK | CPUMCTX_EXTRN_ALL_MSRS);
11905 AssertRCReturn(rc, rc);
11906
11907 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", idMsr, pMixedCtx->edx, pMixedCtx->eax));
11908
11909 VBOXSTRICTRC rcStrict = IEMExecDecodedWrmsr(pVCpu, pVmxTransient->cbInstr);
11910 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11911
11912 if (rcStrict == VINF_SUCCESS)
11913 {
11914 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11915 if ( idMsr == MSR_IA32_APICBASE
11916 || ( idMsr >= MSR_IA32_X2APIC_START
11917 && idMsr <= MSR_IA32_X2APIC_END))
11918 {
11919 /*
11920 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11921 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before IEM changes it.
11922 */
11923 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11924 }
11925 else if (idMsr == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11926 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11927 else if (idMsr == MSR_K6_EFER)
11928 {
11929 /*
11930 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11931 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11932 * the other bits as well, SCE and NXE. See @bugref{7368}.
11933 */
11934 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11935 | HM_CHANGED_VMX_ENTRY_CTLS
11936 | HM_CHANGED_VMX_EXIT_CTLS);
11937 }
11938
11939 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11940 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11941 {
11942 switch (idMsr)
11943 {
11944 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11945 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11946 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11947 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11948 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11949 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11950 default:
11951 {
11952 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11953 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11954 else if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11955 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11956 break;
11957 }
11958 }
11959 }
11960#ifdef VBOX_STRICT
11961 else
11962 {
11963 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11964 switch (idMsr)
11965 {
11966 case MSR_IA32_SYSENTER_CS:
11967 case MSR_IA32_SYSENTER_EIP:
11968 case MSR_IA32_SYSENTER_ESP:
11969 case MSR_K8_FS_BASE:
11970 case MSR_K8_GS_BASE:
11971 {
11972 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", idMsr));
11973 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11974 }
11975
11976 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11977 default:
11978 {
11979 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, idMsr))
11980 {
11981 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11982 if (idMsr != MSR_K6_EFER)
11983 {
11984 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11985 idMsr));
11986 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11987 }
11988 }
11989
11990 if (hmR0VmxIsLazyGuestMsr(pVCpu, idMsr))
11991 {
11992 VMXMSREXITREAD enmRead;
11993 VMXMSREXITWRITE enmWrite;
11994 int rc2 = hmR0VmxGetMsrPermission(pVCpu, idMsr, &enmRead, &enmWrite);
11995 AssertRCReturn(rc2, rc2);
11996 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11997 {
11998 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", idMsr));
11999 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12000 }
12001 }
12002 break;
12003 }
12004 }
12005 }
12006#endif /* VBOX_STRICT */
12007 }
12008 else
12009 AssertMsg( rcStrict == VINF_CPUM_R3_MSR_WRITE
12010 || rcStrict == VINF_IEM_RAISED_XCPT,
12011 ("Unexpected IEMExecDecodedWrmsr status: %Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12012
12013 return rcStrict;
12014}
12015
12016
12017/**
12018 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
12019 */
12020HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12021{
12022 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12023 /** @todo The guest has likely hit a contended spinlock. We might want to
12024 * poke a schedule different guest VCPU. */
12025 return VINF_EM_RAW_INTERRUPT;
12026}
12027
12028
12029/**
12030 * VM-exit handler for when the TPR value is lowered below the specified
12031 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12032 */
12033HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12034{
12035 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12036 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12037
12038 /*
12039 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12040 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12041 */
12042 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12043 return VINF_SUCCESS;
12044}
12045
12046
12047/**
12048 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12049 * VM-exit.
12050 *
12051 * @retval VINF_SUCCESS when guest execution can continue.
12052 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12053 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12054 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12055 * interpreter.
12056 */
12057HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12058{
12059 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12060 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12061
12062 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12063 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12064 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12065 AssertRCReturn(rc, rc);
12066
12067 VBOXSTRICTRC rcStrict;
12068 PVM pVM = pVCpu->CTX_SUFF(pVM);
12069 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12070 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12071 switch (uAccessType)
12072 {
12073 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12074 {
12075 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12076 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12077 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12078 AssertMsg( rcStrict == VINF_SUCCESS
12079 || rcStrict == VINF_IEM_RAISED_XCPT
12080 || rcStrict == VINF_PGM_CHANGE_MODE
12081 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12082
12083 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12084 {
12085 case 0:
12086 {
12087 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12088 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12089 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12090 break;
12091 }
12092
12093 case 2:
12094 {
12095 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12096 /* Nothing to do here, CR2 it's not part of the VMCS. */
12097 break;
12098 }
12099
12100 case 3:
12101 {
12102 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12103 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12104 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12105 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12106 break;
12107 }
12108
12109 case 4:
12110 {
12111 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12113 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12114 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12115 break;
12116 }
12117
12118 case 8:
12119 {
12120 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12121 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12122 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12123 break;
12124 }
12125 default:
12126 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12127 break;
12128 }
12129 break;
12130 }
12131
12132 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12133 {
12134 Assert( !pVM->hm.s.fNestedPaging
12135 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12136 || pVCpu->hm.s.fUsingDebugLoop
12137 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12138 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12139 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12140 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12141
12142 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12143 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12144 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12145 AssertMsg( rcStrict == VINF_SUCCESS
12146 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12147#ifdef VBOX_WITH_STATISTICS
12148 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12149 {
12150 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12151 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12152 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12153 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12154 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12155 }
12156#endif
12157 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12158 VBOXSTRICTRC_VAL(rcStrict)));
12159 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12160 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12161 break;
12162 }
12163
12164 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12165 {
12166 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12167 AssertMsg( rcStrict == VINF_SUCCESS
12168 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12169
12170 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12171 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12172 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12173 break;
12174 }
12175
12176 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12177 {
12178 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12179 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12180 AssertMsg( rcStrict == VINF_SUCCESS
12181 || rcStrict == VINF_IEM_RAISED_XCPT
12182 || rcStrict == VINF_PGM_CHANGE_MODE,
12183 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12184
12185 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12187 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12188 break;
12189 }
12190
12191 default:
12192 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12193 VERR_VMX_UNEXPECTED_EXCEPTION);
12194 }
12195
12196 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12197 : HM_CHANGED_XCPT_RAISED_MASK);
12198 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12199 NOREF(pVM);
12200 return rcStrict;
12201}
12202
12203
12204/**
12205 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12206 * VM-exit.
12207 */
12208HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12209{
12210 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12211 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12212 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12213
12214 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12215 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12216 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12217 | CPUMCTX_EXTRN_SREG_MASK
12218 | CPUMCTX_EXTRN_EFER);
12219 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12220 AssertRCReturn(rc, rc);
12221
12222 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12223 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12224 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12225 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12226 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12227 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12228 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12229 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12230 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12231
12232 /*
12233 * Update exit history to see if this exit can be optimized.
12234 */
12235 VBOXSTRICTRC rcStrict;
12236 PCEMEXITREC pExitRec = NULL;
12237 if ( !fGstStepping
12238 && !fDbgStepping)
12239 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12240 !fIOString
12241 ? !fIOWrite
12242 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12243 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12244 : !fIOWrite
12245 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12246 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12247 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12248 if (!pExitRec)
12249 {
12250 /* I/O operation lookup arrays. */
12251 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12252 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12253 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12254 uint32_t const cbInstr = pVmxTransient->cbInstr;
12255 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12256 PVM pVM = pVCpu->CTX_SUFF(pVM);
12257 if (fIOString)
12258 {
12259 /*
12260 * INS/OUTS - I/O String instruction.
12261 *
12262 * Use instruction-information if available, otherwise fall back on
12263 * interpreting the instruction.
12264 */
12265 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12266 fIOWrite ? 'w' : 'r'));
12267 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12268 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12269 {
12270 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12271 AssertRCReturn(rc2, rc2);
12272 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12273 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12274 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12275 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12276 if (fIOWrite)
12277 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12278 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12279 else
12280 {
12281 /*
12282 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12283 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12284 * See Intel Instruction spec. for "INS".
12285 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12286 */
12287 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12288 }
12289 }
12290 else
12291 rcStrict = IEMExecOne(pVCpu);
12292
12293 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12294 fUpdateRipAlready = true;
12295 }
12296 else
12297 {
12298 /*
12299 * IN/OUT - I/O instruction.
12300 */
12301 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12302 fIOWrite ? 'w' : 'r'));
12303 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12304 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12305 if (fIOWrite)
12306 {
12307 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12309 }
12310 else
12311 {
12312 uint32_t u32Result = 0;
12313 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12314 if (IOM_SUCCESS(rcStrict))
12315 {
12316 /* Save result of I/O IN instr. in AL/AX/EAX. */
12317 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12318 }
12319 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12320 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12321 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12322 }
12323 }
12324
12325 if (IOM_SUCCESS(rcStrict))
12326 {
12327 if (!fUpdateRipAlready)
12328 {
12329 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12330 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12331 }
12332
12333 /*
12334 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12335 * while booting Fedora 17 64-bit guest.
12336 *
12337 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12338 */
12339 if (fIOString)
12340 {
12341 /** @todo Single-step for INS/OUTS with REP prefix? */
12342 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12343 }
12344 else if ( !fDbgStepping
12345 && fGstStepping)
12346 {
12347 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12348 AssertRCReturn(rc, rc);
12349 }
12350
12351 /*
12352 * If any I/O breakpoints are armed, we need to check if one triggered
12353 * and take appropriate action.
12354 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12355 */
12356 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12357 AssertRCReturn(rc, rc);
12358
12359 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12360 * execution engines about whether hyper BPs and such are pending. */
12361 uint32_t const uDr7 = pMixedCtx->dr[7];
12362 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12363 && X86_DR7_ANY_RW_IO(uDr7)
12364 && (pMixedCtx->cr4 & X86_CR4_DE))
12365 || DBGFBpIsHwIoArmed(pVM)))
12366 {
12367 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12368
12369 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12370 VMMRZCallRing3Disable(pVCpu);
12371 HM_DISABLE_PREEMPT();
12372
12373 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12374
12375 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12376 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12377 {
12378 /* Raise #DB. */
12379 if (fIsGuestDbgActive)
12380 ASMSetDR6(pMixedCtx->dr[6]);
12381 if (pMixedCtx->dr[7] != uDr7)
12382 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12383
12384 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12385 }
12386 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12387 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12388 else if ( rcStrict2 != VINF_SUCCESS
12389 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12390 rcStrict = rcStrict2;
12391 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12392
12393 HM_RESTORE_PREEMPT();
12394 VMMRZCallRing3Enable(pVCpu);
12395 }
12396 }
12397
12398#ifdef VBOX_STRICT
12399 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12400 Assert(!fIOWrite);
12401 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12402 Assert(fIOWrite);
12403 else
12404 {
12405# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12406 * statuses, that the VMM device and some others may return. See
12407 * IOM_SUCCESS() for guidance. */
12408 AssertMsg( RT_FAILURE(rcStrict)
12409 || rcStrict == VINF_SUCCESS
12410 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12411 || rcStrict == VINF_EM_DBG_BREAKPOINT
12412 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12413 || rcStrict == VINF_EM_RAW_TO_R3
12414 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12415# endif
12416 }
12417#endif
12418 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12419 }
12420 else
12421 {
12422 /*
12423 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12424 */
12425 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12426 AssertRCReturn(rc2, rc2);
12427 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12428 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12429 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12430 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12431 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12432 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12433
12434 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12435 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12436
12437 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12438 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12439 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12440 }
12441 return rcStrict;
12442}
12443
12444
12445/**
12446 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12447 * VM-exit.
12448 */
12449HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12450{
12451 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12452
12453 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12454 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12455 AssertRCReturn(rc, rc);
12456 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12457 {
12458 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12459 AssertRCReturn(rc, rc);
12460 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12461 {
12462 uint32_t uErrCode;
12463 RTGCUINTPTR GCPtrFaultAddress;
12464 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12465 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12466 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12467 if (fErrorCodeValid)
12468 {
12469 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12470 AssertRCReturn(rc, rc);
12471 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12472 }
12473 else
12474 uErrCode = 0;
12475
12476 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12477 && uVector == X86_XCPT_PF)
12478 GCPtrFaultAddress = pMixedCtx->cr2;
12479 else
12480 GCPtrFaultAddress = 0;
12481
12482 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12483 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12484
12485 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12486 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12487 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12488 }
12489 }
12490
12491 /* Fall back to the interpreter to emulate the task-switch. */
12492 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12493 return VERR_EM_INTERPRETER;
12494}
12495
12496
12497/**
12498 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12499 */
12500HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12501{
12502 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12503 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12504 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12505 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12506 AssertRCReturn(rc, rc);
12507 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12508 return VINF_EM_DBG_STEPPED;
12509}
12510
12511
12512/**
12513 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12514 */
12515HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12516{
12517 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12518
12519 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12520
12521 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12522 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12523 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12524 {
12525 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12526 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12527 {
12528 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12529 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12530 }
12531 }
12532 else
12533 {
12534 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12535 rcStrict1 = VINF_SUCCESS;
12536 return rcStrict1;
12537 }
12538
12539 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12540 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12541 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12542 AssertRCReturn(rc, rc);
12543
12544 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12545 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12546 VBOXSTRICTRC rcStrict2;
12547 switch (uAccessType)
12548 {
12549 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12550 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12551 {
12552 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12553 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12554 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12555
12556 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12557 GCPhys &= PAGE_BASE_GC_MASK;
12558 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12559 PVM pVM = pVCpu->CTX_SUFF(pVM);
12560 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12561 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12562
12563 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12564 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12565 CPUMCTX2CORE(pMixedCtx), GCPhys);
12566 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12567 if ( rcStrict2 == VINF_SUCCESS
12568 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12569 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12570 {
12571 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12572 | HM_CHANGED_GUEST_RSP
12573 | HM_CHANGED_GUEST_RFLAGS
12574 | HM_CHANGED_GUEST_APIC_TPR);
12575 rcStrict2 = VINF_SUCCESS;
12576 }
12577 break;
12578 }
12579
12580 default:
12581 Log4Func(("uAccessType=%#x\n", uAccessType));
12582 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12583 break;
12584 }
12585
12586 if (rcStrict2 != VINF_SUCCESS)
12587 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12588 return rcStrict2;
12589}
12590
12591
12592/**
12593 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12594 * VM-exit.
12595 */
12596HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12597{
12598 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12599
12600 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12601 if (pVmxTransient->fWasGuestDebugStateActive)
12602 {
12603 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12604 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12605 }
12606
12607 if ( !pVCpu->hm.s.fSingleInstruction
12608 && !pVmxTransient->fWasHyperDebugStateActive)
12609 {
12610 Assert(!DBGFIsStepping(pVCpu));
12611 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12612
12613 /* Don't intercept MOV DRx any more. */
12614 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12615 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12616 AssertRCReturn(rc, rc);
12617
12618 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12619 VMMRZCallRing3Disable(pVCpu);
12620 HM_DISABLE_PREEMPT();
12621
12622 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12623 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12624 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12625
12626 HM_RESTORE_PREEMPT();
12627 VMMRZCallRing3Enable(pVCpu);
12628
12629#ifdef VBOX_WITH_STATISTICS
12630 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12631 AssertRCReturn(rc, rc);
12632 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12634 else
12635 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12636#endif
12637 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12638 return VINF_SUCCESS;
12639 }
12640
12641 /*
12642 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12643 * Update the segment registers and DR7 from the CPU.
12644 */
12645 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12646 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12647 | CPUMCTX_EXTRN_DR7);
12648 AssertRCReturn(rc, rc);
12649 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12650
12651 PVM pVM = pVCpu->CTX_SUFF(pVM);
12652 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12653 {
12654 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12655 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12656 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12657 if (RT_SUCCESS(rc))
12658 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12659 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12660 }
12661 else
12662 {
12663 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12664 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12665 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12666 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12667 }
12668
12669 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12670 if (RT_SUCCESS(rc))
12671 {
12672 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12673 AssertRCReturn(rc2, rc2);
12674 return VINF_SUCCESS;
12675 }
12676 return rc;
12677}
12678
12679
12680/**
12681 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12682 * Conditional VM-exit.
12683 */
12684HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12685{
12686 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12687 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12688
12689 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12690 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12691 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12692 {
12693 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12694 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12695 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12696 {
12697 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12698 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12699 }
12700 }
12701 else
12702 {
12703 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12704 rcStrict1 = VINF_SUCCESS;
12705 return rcStrict1;
12706 }
12707
12708 /*
12709 * Get sufficent state and update the exit history entry.
12710 */
12711 RTGCPHYS GCPhys;
12712 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12713 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12714 AssertRCReturn(rc, rc);
12715
12716 VBOXSTRICTRC rcStrict;
12717 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12718 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12719 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12720 if (!pExitRec)
12721 {
12722 /*
12723 * If we succeed, resume guest execution.
12724 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12725 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12726 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12727 * weird case. See @bugref{6043}.
12728 */
12729 PVM pVM = pVCpu->CTX_SUFF(pVM);
12730 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12731 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12732 if ( rcStrict == VINF_SUCCESS
12733 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12734 || rcStrict == VERR_PAGE_NOT_PRESENT)
12735 {
12736 /* Successfully handled MMIO operation. */
12737 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12738 | HM_CHANGED_GUEST_RSP
12739 | HM_CHANGED_GUEST_RFLAGS
12740 | HM_CHANGED_GUEST_APIC_TPR);
12741 rcStrict = VINF_SUCCESS;
12742 }
12743 }
12744 else
12745 {
12746 /*
12747 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12748 */
12749 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12750 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12751 AssertRCReturn(rc2, rc2);
12752
12753 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12754 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12755
12756 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12757 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12758
12759 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12760 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12761 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12762 }
12763 return VBOXSTRICTRC_TODO(rcStrict);
12764}
12765
12766
12767/**
12768 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12769 * VM-exit.
12770 */
12771HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12772{
12773 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12774 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12775
12776 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12777 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12778 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12779 {
12780 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12781 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12782 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12783 }
12784 else
12785 {
12786 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12787 rcStrict1 = VINF_SUCCESS;
12788 return rcStrict1;
12789 }
12790
12791 RTGCPHYS GCPhys;
12792 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12793 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12794 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12795 AssertRCReturn(rc, rc);
12796
12797 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12798 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12799
12800 RTGCUINT uErrorCode = 0;
12801 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12802 uErrorCode |= X86_TRAP_PF_ID;
12803 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12804 uErrorCode |= X86_TRAP_PF_RW;
12805 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12806 uErrorCode |= X86_TRAP_PF_P;
12807
12808 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12809
12810 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12811 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12812
12813 /* Handle the pagefault trap for the nested shadow table. */
12814 PVM pVM = pVCpu->CTX_SUFF(pVM);
12815 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12816 TRPMResetTrap(pVCpu);
12817
12818 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12819 if ( rcStrict2 == VINF_SUCCESS
12820 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12821 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12822 {
12823 /* Successfully synced our nested page tables. */
12824 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12825 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12826 | HM_CHANGED_GUEST_RSP
12827 | HM_CHANGED_GUEST_RFLAGS);
12828 return VINF_SUCCESS;
12829 }
12830
12831 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12832 return rcStrict2;
12833}
12834
12835/** @} */
12836
12837/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12838/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12839/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12840
12841/** @name VM-exit exception handlers.
12842 * @{
12843 */
12844
12845/**
12846 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12847 */
12848static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12849{
12850 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12851 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12852
12853 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12854 AssertRCReturn(rc, rc);
12855
12856 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12857 {
12858 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12859 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12860
12861 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12862 * provides VM-exit instruction length. If this causes problem later,
12863 * disassemble the instruction like it's done on AMD-V. */
12864 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12865 AssertRCReturn(rc2, rc2);
12866 return rc;
12867 }
12868
12869 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12870 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12871 return rc;
12872}
12873
12874
12875/**
12876 * VM-exit exception handler for \#BP (Breakpoint exception).
12877 */
12878static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12879{
12880 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12881 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12882
12883 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12884 AssertRCReturn(rc, rc);
12885
12886 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12887 if (rc == VINF_EM_RAW_GUEST_TRAP)
12888 {
12889 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12890 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12891 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12892 AssertRCReturn(rc, rc);
12893
12894 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12895 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12896 }
12897
12898 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12899 return rc;
12900}
12901
12902
12903/**
12904 * VM-exit exception handler for \#AC (alignment check exception).
12905 */
12906static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12907{
12908 RT_NOREF_PV(pMixedCtx);
12909 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12910
12911 /*
12912 * Re-inject it. We'll detect any nesting before getting here.
12913 */
12914 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12915 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12916 AssertRCReturn(rc, rc);
12917 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12918
12919 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12920 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12921 return VINF_SUCCESS;
12922}
12923
12924
12925/**
12926 * VM-exit exception handler for \#DB (Debug exception).
12927 */
12928static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12929{
12930 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12931 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12932
12933 /*
12934 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12935 * for processing.
12936 */
12937 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12938
12939 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12940 uint64_t uDR6 = X86_DR6_INIT_VAL;
12941 uDR6 |= ( pVmxTransient->uExitQualification
12942 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12943
12944 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12945 Log6Func(("rc=%Rrc\n", rc));
12946 if (rc == VINF_EM_RAW_GUEST_TRAP)
12947 {
12948 /*
12949 * The exception was for the guest. Update DR6, DR7.GD and
12950 * IA32_DEBUGCTL.LBR before forwarding it.
12951 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12952 */
12953 VMMRZCallRing3Disable(pVCpu);
12954 HM_DISABLE_PREEMPT();
12955
12956 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12957 pMixedCtx->dr[6] |= uDR6;
12958 if (CPUMIsGuestDebugStateActive(pVCpu))
12959 ASMSetDR6(pMixedCtx->dr[6]);
12960
12961 HM_RESTORE_PREEMPT();
12962 VMMRZCallRing3Enable(pVCpu);
12963
12964 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12965 AssertRCReturn(rc, rc);
12966
12967 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12968 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12969
12970 /* Paranoia. */
12971 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12972 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12973
12974 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12975 AssertRCReturn(rc, rc);
12976
12977 /*
12978 * Raise #DB in the guest.
12979 *
12980 * It is important to reflect exactly what the VM-exit gave us (preserving the
12981 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12982 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12983 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12984 *
12985 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12986 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12987 */
12988 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12989 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12990 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12991 AssertRCReturn(rc, rc);
12992 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12993 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12994 return VINF_SUCCESS;
12995 }
12996
12997 /*
12998 * Not a guest trap, must be a hypervisor related debug event then.
12999 * Update DR6 in case someone is interested in it.
13000 */
13001 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
13002 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
13003 CPUMSetHyperDR6(pVCpu, uDR6);
13004
13005 return rc;
13006}
13007
13008/**
13009 * VM-exit exception handler for \#GP (General-protection exception).
13010 *
13011 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
13012 */
13013static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13014{
13015 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
13017
13018 int rc;
13019 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
13020 { /* likely */ }
13021 else
13022 {
13023#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13024 Assert(pVCpu->hm.s.fUsingDebugLoop);
13025#endif
13026 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13027 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13028 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13029 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13030 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13031 AssertRCReturn(rc, rc);
13032 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13033 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13034 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13035 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13036 return rc;
13037 }
13038
13039 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13040 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13041
13042 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13043 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13044 AssertRCReturn(rc, rc);
13045
13046 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13047 uint32_t cbOp = 0;
13048 PVM pVM = pVCpu->CTX_SUFF(pVM);
13049 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13050 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13051 if (RT_SUCCESS(rc))
13052 {
13053 rc = VINF_SUCCESS;
13054 Assert(cbOp == pDis->cbInstr);
13055 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13056 switch (pDis->pCurInstr->uOpcode)
13057 {
13058 case OP_CLI:
13059 {
13060 pMixedCtx->eflags.Bits.u1IF = 0;
13061 pMixedCtx->eflags.Bits.u1RF = 0;
13062 pMixedCtx->rip += pDis->cbInstr;
13063 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13064 if ( !fDbgStepping
13065 && pMixedCtx->eflags.Bits.u1TF)
13066 {
13067 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13068 AssertRCReturn(rc, rc);
13069 }
13070 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13071 break;
13072 }
13073
13074 case OP_STI:
13075 {
13076 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13077 pMixedCtx->eflags.Bits.u1IF = 1;
13078 pMixedCtx->eflags.Bits.u1RF = 0;
13079 pMixedCtx->rip += pDis->cbInstr;
13080 if (!fOldIF)
13081 {
13082 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13083 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13084 }
13085 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13086 if ( !fDbgStepping
13087 && pMixedCtx->eflags.Bits.u1TF)
13088 {
13089 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13090 AssertRCReturn(rc, rc);
13091 }
13092 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13093 break;
13094 }
13095
13096 case OP_HLT:
13097 {
13098 rc = VINF_EM_HALT;
13099 pMixedCtx->rip += pDis->cbInstr;
13100 pMixedCtx->eflags.Bits.u1RF = 0;
13101 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13102 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13103 break;
13104 }
13105
13106 case OP_POPF:
13107 {
13108 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13109 uint32_t cbParm;
13110 uint32_t uMask;
13111 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13112 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13113 {
13114 cbParm = 4;
13115 uMask = 0xffffffff;
13116 }
13117 else
13118 {
13119 cbParm = 2;
13120 uMask = 0xffff;
13121 }
13122
13123 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13124 RTGCPTR GCPtrStack = 0;
13125 X86EFLAGS Eflags;
13126 Eflags.u32 = 0;
13127 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13128 &GCPtrStack);
13129 if (RT_SUCCESS(rc))
13130 {
13131 Assert(sizeof(Eflags.u32) >= cbParm);
13132 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13133 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13134 }
13135 if (RT_FAILURE(rc))
13136 {
13137 rc = VERR_EM_INTERPRETER;
13138 break;
13139 }
13140 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13141 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13142 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13143 pMixedCtx->esp += cbParm;
13144 pMixedCtx->esp &= uMask;
13145 pMixedCtx->rip += pDis->cbInstr;
13146 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13147 | HM_CHANGED_GUEST_RSP
13148 | HM_CHANGED_GUEST_RFLAGS);
13149 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13150 POPF restores EFLAGS.TF. */
13151 if ( !fDbgStepping
13152 && fGstStepping)
13153 {
13154 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13155 AssertRCReturn(rc, rc);
13156 }
13157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13158 break;
13159 }
13160
13161 case OP_PUSHF:
13162 {
13163 uint32_t cbParm;
13164 uint32_t uMask;
13165 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13166 {
13167 cbParm = 4;
13168 uMask = 0xffffffff;
13169 }
13170 else
13171 {
13172 cbParm = 2;
13173 uMask = 0xffff;
13174 }
13175
13176 /* Get the stack pointer & push the contents of eflags onto the stack. */
13177 RTGCPTR GCPtrStack = 0;
13178 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13179 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13180 if (RT_FAILURE(rc))
13181 {
13182 rc = VERR_EM_INTERPRETER;
13183 break;
13184 }
13185 X86EFLAGS Eflags = pMixedCtx->eflags;
13186 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13187 Eflags.Bits.u1RF = 0;
13188 Eflags.Bits.u1VM = 0;
13189
13190 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13191 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13192 {
13193 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13194 rc = VERR_EM_INTERPRETER;
13195 break;
13196 }
13197 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13198 pMixedCtx->esp -= cbParm;
13199 pMixedCtx->esp &= uMask;
13200 pMixedCtx->rip += pDis->cbInstr;
13201 pMixedCtx->eflags.Bits.u1RF = 0;
13202 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13203 | HM_CHANGED_GUEST_RSP
13204 | HM_CHANGED_GUEST_RFLAGS);
13205 if ( !fDbgStepping
13206 && pMixedCtx->eflags.Bits.u1TF)
13207 {
13208 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13209 AssertRCReturn(rc, rc);
13210 }
13211 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13212 break;
13213 }
13214
13215 case OP_IRET:
13216 {
13217 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13218 * instruction reference. */
13219 RTGCPTR GCPtrStack = 0;
13220 uint32_t uMask = 0xffff;
13221 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13222 uint16_t aIretFrame[3];
13223 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13224 {
13225 rc = VERR_EM_INTERPRETER;
13226 break;
13227 }
13228 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13229 &GCPtrStack);
13230 if (RT_SUCCESS(rc))
13231 {
13232 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13233 PGMACCESSORIGIN_HM));
13234 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13235 }
13236 if (RT_FAILURE(rc))
13237 {
13238 rc = VERR_EM_INTERPRETER;
13239 break;
13240 }
13241 pMixedCtx->eip = 0;
13242 pMixedCtx->ip = aIretFrame[0];
13243 pMixedCtx->cs.Sel = aIretFrame[1];
13244 pMixedCtx->cs.ValidSel = aIretFrame[1];
13245 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13246 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13247 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13248 pMixedCtx->sp += sizeof(aIretFrame);
13249 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13250 | HM_CHANGED_GUEST_CS
13251 | HM_CHANGED_GUEST_RSP
13252 | HM_CHANGED_GUEST_RFLAGS);
13253 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13254 if ( !fDbgStepping
13255 && fGstStepping)
13256 {
13257 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13258 AssertRCReturn(rc, rc);
13259 }
13260 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13261 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13262 break;
13263 }
13264
13265 case OP_INT:
13266 {
13267 uint16_t uVector = pDis->Param1.uValue & 0xff;
13268 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13269 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13270 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13271 break;
13272 }
13273
13274 case OP_INTO:
13275 {
13276 if (pMixedCtx->eflags.Bits.u1OF)
13277 {
13278 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13279 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13280 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13281 }
13282 else
13283 {
13284 pMixedCtx->eflags.Bits.u1RF = 0;
13285 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13286 }
13287 break;
13288 }
13289
13290 default:
13291 {
13292 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13293 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13294 EMCODETYPE_SUPERVISOR);
13295 rc = VBOXSTRICTRC_VAL(rc2);
13296 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13297 /** @todo We have to set pending-debug exceptions here when the guest is
13298 * single-stepping depending on the instruction that was interpreted. */
13299 Log4Func(("#GP rc=%Rrc\n", rc));
13300 break;
13301 }
13302 }
13303 }
13304 else
13305 rc = VERR_EM_INTERPRETER;
13306
13307 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13308 ("#GP Unexpected rc=%Rrc\n", rc));
13309 return rc;
13310}
13311
13312
13313/**
13314 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13315 * the exception reported in the VMX transient structure back into the VM.
13316 *
13317 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13318 * up-to-date.
13319 */
13320static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13321{
13322 RT_NOREF_PV(pMixedCtx);
13323 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13324#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13325 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13326 ("uVector=%#x u32XcptBitmap=%#X32\n",
13327 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13328#endif
13329
13330 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13331 hmR0VmxCheckExitDueToEventDelivery(). */
13332 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13333 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13334 AssertRCReturn(rc, rc);
13335 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13336
13337#ifdef DEBUG_ramshankar
13338 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13339 | CPUMCTX_EXTRN_RIP);
13340 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13341 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13342#endif
13343
13344 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13345 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13346 return VINF_SUCCESS;
13347}
13348
13349
13350/**
13351 * VM-exit exception handler for \#PF (Page-fault exception).
13352 */
13353static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13354{
13355 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13356 PVM pVM = pVCpu->CTX_SUFF(pVM);
13357 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13358 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13359 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13360 AssertRCReturn(rc, rc);
13361
13362 if (!pVM->hm.s.fNestedPaging)
13363 { /* likely */ }
13364 else
13365 {
13366#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13367 Assert(pVCpu->hm.s.fUsingDebugLoop);
13368#endif
13369 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13370 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13371 {
13372 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13373 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13374 }
13375 else
13376 {
13377 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13378 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13379 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13380 }
13381 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13382 return rc;
13383 }
13384
13385 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13386 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13387 if (pVmxTransient->fVectoringPF)
13388 {
13389 Assert(pVCpu->hm.s.Event.fPending);
13390 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13391 }
13392
13393 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13394 AssertRCReturn(rc, rc);
13395
13396 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13397 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13398
13399 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13400 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13401 (RTGCPTR)pVmxTransient->uExitQualification);
13402
13403 Log4Func(("#PF: rc=%Rrc\n", rc));
13404 if (rc == VINF_SUCCESS)
13405 {
13406 /*
13407 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13408 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13409 */
13410 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13411 TRPMResetTrap(pVCpu);
13412 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13413 return rc;
13414 }
13415
13416 if (rc == VINF_EM_RAW_GUEST_TRAP)
13417 {
13418 if (!pVmxTransient->fVectoringDoublePF)
13419 {
13420 /* It's a guest page fault and needs to be reflected to the guest. */
13421 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13422 TRPMResetTrap(pVCpu);
13423 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13424 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13425 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13426 }
13427 else
13428 {
13429 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13430 TRPMResetTrap(pVCpu);
13431 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13432 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13433 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13434 }
13435
13436 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13437 return VINF_SUCCESS;
13438 }
13439
13440 TRPMResetTrap(pVCpu);
13441 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13442 return rc;
13443}
13444
13445/** @} */
13446
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette