VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72851

Last change on this file since 72851 was 72851, checked in by vboxsync, 7 years ago

VMM/HMVMXR0: Macro nit, try not to hide parameters.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.8 KB
Line 
1/* $Id: HMVMXR0.cpp 72851 2018-07-04 05:36:58Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
152 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162/** Helper macro for VM-exit handlers called unexpectedly. */
163#define HMVMX_UNEXPECTED_EXIT_RET(a_pVCpu, a_pVmxTransient) \
164 do { \
165 (a_pVCpu)->hm.s.u32HMError = (a_pVmxTransient)->uExitReason; \
166 return VERR_VMX_UNEXPECTED_EXIT; \
167 } while (0)
168
169/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
170#ifdef VMX_USE_CACHED_VMCS_ACCESSES
171# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
172 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
173 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
174#else
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#endif
179
180/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
181# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
182 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
183 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
184
185
186/*********************************************************************************************************************************
187* Structures and Typedefs *
188*********************************************************************************************************************************/
189/**
190 * VMX transient state.
191 *
192 * A state structure for holding miscellaneous information across
193 * VMX non-root operation and restored after the transition.
194 */
195typedef struct VMXTRANSIENT
196{
197 /** The host's rflags/eflags. */
198 RTCCUINTREG fEFlags;
199#if HC_ARCH_BITS == 32
200 uint32_t u32Alignment0;
201#endif
202 /** The guest's TPR value used for TPR shadowing. */
203 uint8_t u8GuestTpr;
204 /** Alignment. */
205 uint8_t abAlignment0[7];
206
207 /** The basic VM-exit reason. */
208 uint16_t uExitReason;
209 /** Alignment. */
210 uint16_t u16Alignment0;
211 /** The VM-exit interruption error code. */
212 uint32_t uExitIntErrorCode;
213 /** The VM-exit exit code qualification. */
214 uint64_t uExitQualification;
215
216 /** The VM-exit interruption-information field. */
217 uint32_t uExitIntInfo;
218 /** The VM-exit instruction-length field. */
219 uint32_t cbInstr;
220 /** The VM-exit instruction-information field. */
221 union
222 {
223 /** Plain unsigned int representation. */
224 uint32_t u;
225 /** INS and OUTS information. */
226 struct
227 {
228 uint32_t u7Reserved0 : 7;
229 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
230 uint32_t u3AddrSize : 3;
231 uint32_t u5Reserved1 : 5;
232 /** The segment register (X86_SREG_XXX). */
233 uint32_t iSegReg : 3;
234 uint32_t uReserved2 : 14;
235 } StrIo;
236 /** INVEPT, INVVPID, INVPCID information. */
237 struct
238 {
239 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
240 uint32_t u2Scaling : 2;
241 uint32_t u5Reserved0 : 5;
242 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
243 uint32_t u3AddrSize : 3;
244 uint32_t u1Reserved0 : 1;
245 uint32_t u4Reserved0 : 4;
246 /** The segment register (X86_SREG_XXX). */
247 uint32_t iSegReg : 3;
248 /** The index register (X86_GREG_XXX). */
249 uint32_t iIdxReg : 4;
250 /** Set if index register is invalid. */
251 uint32_t fIdxRegValid : 1;
252 /** The base register (X86_GREG_XXX). */
253 uint32_t iBaseReg : 4;
254 /** Set if base register is invalid. */
255 uint32_t fBaseRegValid : 1;
256 /** Register 2 (X86_GREG_XXX). */
257 uint32_t iReg2 : 4;
258 } Inv;
259 } ExitInstrInfo;
260 /** Whether the VM-entry failed or not. */
261 bool fVMEntryFailed;
262 /** Alignment. */
263 uint8_t abAlignment1[3];
264
265 /** The VM-entry interruption-information field. */
266 uint32_t uEntryIntInfo;
267 /** The VM-entry exception error code field. */
268 uint32_t uEntryXcptErrorCode;
269 /** The VM-entry instruction length field. */
270 uint32_t cbEntryInstr;
271
272 /** IDT-vectoring information field. */
273 uint32_t uIdtVectoringInfo;
274 /** IDT-vectoring error code. */
275 uint32_t uIdtVectoringErrorCode;
276
277 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
278 uint32_t fVmcsFieldsRead;
279
280 /** Whether the guest debug state was active at the time of VM-exit. */
281 bool fWasGuestDebugStateActive;
282 /** Whether the hyper debug state was active at the time of VM-exit. */
283 bool fWasHyperDebugStateActive;
284 /** Whether TSC-offsetting should be setup before VM-entry. */
285 bool fUpdateTscOffsettingAndPreemptTimer;
286 /** Whether the VM-exit was caused by a page-fault during delivery of a
287 * contributory exception or a page-fault. */
288 bool fVectoringDoublePF;
289 /** Whether the VM-exit was caused by a page-fault during delivery of an
290 * external interrupt or NMI. */
291 bool fVectoringPF;
292} VMXTRANSIENT;
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
297AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
298/** Pointer to VMX transient state. */
299typedef VMXTRANSIENT *PVMXTRANSIENT;
300
301
302/**
303 * MSR-bitmap read permissions.
304 */
305typedef enum VMXMSREXITREAD
306{
307 /** Reading this MSR causes a VM-exit. */
308 VMXMSREXIT_INTERCEPT_READ = 0xb,
309 /** Reading this MSR does not cause a VM-exit. */
310 VMXMSREXIT_PASSTHRU_READ
311} VMXMSREXITREAD;
312/** Pointer to MSR-bitmap read permissions. */
313typedef VMXMSREXITREAD* PVMXMSREXITREAD;
314
315/**
316 * MSR-bitmap write permissions.
317 */
318typedef enum VMXMSREXITWRITE
319{
320 /** Writing to this MSR causes a VM-exit. */
321 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
322 /** Writing to this MSR does not cause a VM-exit. */
323 VMXMSREXIT_PASSTHRU_WRITE
324} VMXMSREXITWRITE;
325/** Pointer to MSR-bitmap write permissions. */
326typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
327
328
329/**
330 * VMX VM-exit handler.
331 *
332 * @returns Strict VBox status code (i.e. informational status codes too).
333 * @param pVCpu The cross context virtual CPU structure.
334 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
335 * out-of-sync. Make sure to update the required
336 * fields before using them.
337 * @param pVmxTransient Pointer to the VMX-transient structure.
338 */
339#ifndef HMVMX_USE_FUNCTION_TABLE
340typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
341#else
342typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
343/** Pointer to VM-exit handler. */
344typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
345#endif
346
347/**
348 * VMX VM-exit handler, non-strict status code.
349 *
350 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
351 *
352 * @returns VBox status code, no informational status code returned.
353 * @param pVCpu The cross context virtual CPU structure.
354 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
355 * out-of-sync. Make sure to update the required
356 * fields before using them.
357 * @param pVmxTransient Pointer to the VMX-transient structure.
358 *
359 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
360 * use of that status code will be replaced with VINF_EM_SOMETHING
361 * later when switching over to IEM.
362 */
363#ifndef HMVMX_USE_FUNCTION_TABLE
364typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
365#else
366typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
367#endif
368
369
370/*********************************************************************************************************************************
371* Internal Functions *
372*********************************************************************************************************************************/
373static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
374static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
375static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
376static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
377static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
378 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
379#if HC_ARCH_BITS == 32
380static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
381#endif
382#ifndef HMVMX_USE_FUNCTION_TABLE
383DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
384# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
385# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
386#else
387# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
388# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
389#endif
390
391
392/** @name VM-exit handlers.
393 * @{
394 */
395static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
396static FNVMXEXITHANDLER hmR0VmxExitExtInt;
397static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
404static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
405static FNVMXEXITHANDLER hmR0VmxExitCpuid;
406static FNVMXEXITHANDLER hmR0VmxExitGetsec;
407static FNVMXEXITHANDLER hmR0VmxExitHlt;
408static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
409static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
410static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
411static FNVMXEXITHANDLER hmR0VmxExitVmcall;
412static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
415static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
416static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
417static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
418static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
419static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
423static FNVMXEXITHANDLER hmR0VmxExitMwait;
424static FNVMXEXITHANDLER hmR0VmxExitMtf;
425static FNVMXEXITHANDLER hmR0VmxExitMonitor;
426static FNVMXEXITHANDLER hmR0VmxExitPause;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
429static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
432static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
433static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
434static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
435static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
436static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
437static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
438static FNVMXEXITHANDLER hmR0VmxExitRdrand;
439static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
440/** @} */
441
442static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
450
451
452/*********************************************************************************************************************************
453* Global Variables *
454*********************************************************************************************************************************/
455#ifdef HMVMX_USE_FUNCTION_TABLE
456
457/**
458 * VMX_EXIT dispatch table.
459 */
460static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
461{
462 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
463 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
464 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
465 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
466 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
467 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
468 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
469 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
470 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
471 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
472 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
473 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
474 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
475 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
476 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
477 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
478 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
479 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
480 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
481 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
482 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
483 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
484 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
485 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
486 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
487 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
488 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
489 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
490 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
491 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
492 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
493 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
494 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
495 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
496 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
497 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
498 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
499 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
500 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
501 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
502 /* 40 UNDEFINED */ hmR0VmxExitPause,
503 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
504 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
505 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
506 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
507 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
508 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
510 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
511 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
512 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
513 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
514 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
515 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
516 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
517 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
518 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
519 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
520 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
521 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
522 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
523 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
524 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
525 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
526 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
527};
528#endif /* HMVMX_USE_FUNCTION_TABLE */
529
530#ifdef VBOX_STRICT
531static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
532{
533 /* 0 */ "(Not Used)",
534 /* 1 */ "VMCALL executed in VMX root operation.",
535 /* 2 */ "VMCLEAR with invalid physical address.",
536 /* 3 */ "VMCLEAR with VMXON pointer.",
537 /* 4 */ "VMLAUNCH with non-clear VMCS.",
538 /* 5 */ "VMRESUME with non-launched VMCS.",
539 /* 6 */ "VMRESUME after VMXOFF",
540 /* 7 */ "VM-entry with invalid control fields.",
541 /* 8 */ "VM-entry with invalid host state fields.",
542 /* 9 */ "VMPTRLD with invalid physical address.",
543 /* 10 */ "VMPTRLD with VMXON pointer.",
544 /* 11 */ "VMPTRLD with incorrect revision identifier.",
545 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
546 /* 13 */ "VMWRITE to read-only VMCS component.",
547 /* 14 */ "(Not Used)",
548 /* 15 */ "VMXON executed in VMX root operation.",
549 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
550 /* 17 */ "VM-entry with non-launched executing VMCS.",
551 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
552 /* 19 */ "VMCALL with non-clear VMCS.",
553 /* 20 */ "VMCALL with invalid VM-exit control fields.",
554 /* 21 */ "(Not Used)",
555 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
556 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
557 /* 24 */ "VMCALL with invalid SMM-monitor features.",
558 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
559 /* 26 */ "VM-entry with events blocked by MOV SS.",
560 /* 27 */ "(Not Used)",
561 /* 28 */ "Invalid operand to INVEPT/INVVPID."
562};
563#endif /* VBOX_STRICT */
564
565
566
567/**
568 * Updates the VM's last error record.
569 *
570 * If there was a VMX instruction error, reads the error data from the VMCS and
571 * updates VCPU's last error record as well.
572 *
573 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
574 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
575 * VERR_VMX_INVALID_VMCS_FIELD.
576 * @param rc The error code.
577 */
578static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
579{
580 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
581 || rc == VERR_VMX_UNABLE_TO_START_VM)
582 {
583 AssertPtrReturnVoid(pVCpu);
584 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
585 }
586 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
587}
588
589
590/**
591 * Reads the VM-entry interruption-information field from the VMCS into the VMX
592 * transient structure.
593 *
594 * @returns VBox status code.
595 * @param pVmxTransient Pointer to the VMX transient structure.
596 *
597 * @remarks No-long-jump zone!!!
598 */
599DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
600{
601 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
602 AssertRCReturn(rc, rc);
603 return VINF_SUCCESS;
604}
605
606#ifdef VBOX_STRICT
607/**
608 * Reads the VM-entry exception error code field from the VMCS into
609 * the VMX transient structure.
610 *
611 * @returns VBox status code.
612 * @param pVmxTransient Pointer to the VMX transient structure.
613 *
614 * @remarks No-long-jump zone!!!
615 */
616DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
617{
618 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
619 AssertRCReturn(rc, rc);
620 return VINF_SUCCESS;
621}
622
623
624/**
625 * Reads the VM-entry exception error code field from the VMCS into
626 * the VMX transient structure.
627 *
628 * @returns VBox status code.
629 * @param pVmxTransient Pointer to the VMX transient structure.
630 *
631 * @remarks No-long-jump zone!!!
632 */
633DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
634{
635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
636 AssertRCReturn(rc, rc);
637 return VINF_SUCCESS;
638}
639#endif /* VBOX_STRICT */
640
641
642/**
643 * Reads the VM-exit interruption-information field from the VMCS into the VMX
644 * transient structure.
645 *
646 * @returns VBox status code.
647 * @param pVmxTransient Pointer to the VMX transient structure.
648 */
649DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
650{
651 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
652 {
653 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
654 AssertRCReturn(rc,rc);
655 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
656 }
657 return VINF_SUCCESS;
658}
659
660
661/**
662 * Reads the VM-exit interruption error code from the VMCS into the VMX
663 * transient structure.
664 *
665 * @returns VBox status code.
666 * @param pVmxTransient Pointer to the VMX transient structure.
667 */
668DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
669{
670 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
671 {
672 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
673 AssertRCReturn(rc, rc);
674 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
675 }
676 return VINF_SUCCESS;
677}
678
679
680/**
681 * Reads the VM-exit instruction length field from the VMCS into the VMX
682 * transient structure.
683 *
684 * @returns VBox status code.
685 * @param pVmxTransient Pointer to the VMX transient structure.
686 */
687DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
688{
689 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
690 {
691 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
692 AssertRCReturn(rc, rc);
693 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
694 }
695 return VINF_SUCCESS;
696}
697
698
699/**
700 * Reads the VM-exit instruction-information field from the VMCS into
701 * the VMX transient structure.
702 *
703 * @returns VBox status code.
704 * @param pVmxTransient Pointer to the VMX transient structure.
705 */
706DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
707{
708 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
709 {
710 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
711 AssertRCReturn(rc, rc);
712 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
713 }
714 return VINF_SUCCESS;
715}
716
717
718/**
719 * Reads the exit code qualification from the VMCS into the VMX transient
720 * structure.
721 *
722 * @returns VBox status code.
723 * @param pVCpu The cross context virtual CPU structure of the
724 * calling EMT. (Required for the VMCS cache case.)
725 * @param pVmxTransient Pointer to the VMX transient structure.
726 */
727DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
728{
729 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
730 {
731 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
732 AssertRCReturn(rc, rc);
733 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
734 }
735 return VINF_SUCCESS;
736}
737
738
739/**
740 * Reads the IDT-vectoring information field from the VMCS into the VMX
741 * transient structure.
742 *
743 * @returns VBox status code.
744 * @param pVmxTransient Pointer to the VMX transient structure.
745 *
746 * @remarks No-long-jump zone!!!
747 */
748DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
749{
750 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
751 {
752 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
753 AssertRCReturn(rc, rc);
754 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
755 }
756 return VINF_SUCCESS;
757}
758
759
760/**
761 * Reads the IDT-vectoring error code from the VMCS into the VMX
762 * transient structure.
763 *
764 * @returns VBox status code.
765 * @param pVmxTransient Pointer to the VMX transient structure.
766 */
767DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
768{
769 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
770 {
771 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
772 AssertRCReturn(rc, rc);
773 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
774 }
775 return VINF_SUCCESS;
776}
777
778
779/**
780 * Enters VMX root mode operation on the current CPU.
781 *
782 * @returns VBox status code.
783 * @param pVM The cross context VM structure. Can be
784 * NULL, after a resume.
785 * @param HCPhysCpuPage Physical address of the VMXON region.
786 * @param pvCpuPage Pointer to the VMXON region.
787 */
788static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
789{
790 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
791 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
792 Assert(pvCpuPage);
793 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
794
795 if (pVM)
796 {
797 /* Write the VMCS revision dword to the VMXON region. */
798 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
799 }
800
801 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
802 RTCCUINTREG fEFlags = ASMIntDisableFlags();
803
804 /* Enable the VMX bit in CR4 if necessary. */
805 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
806
807 /* Enter VMX root mode. */
808 int rc = VMXEnable(HCPhysCpuPage);
809 if (RT_FAILURE(rc))
810 {
811 if (!(uOldCr4 & X86_CR4_VMXE))
812 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
813
814 if (pVM)
815 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
816 }
817
818 /* Restore interrupts. */
819 ASMSetFlags(fEFlags);
820 return rc;
821}
822
823
824/**
825 * Exits VMX root mode operation on the current CPU.
826 *
827 * @returns VBox status code.
828 */
829static int hmR0VmxLeaveRootMode(void)
830{
831 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
832
833 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
834 RTCCUINTREG fEFlags = ASMIntDisableFlags();
835
836 /* If we're for some reason not in VMX root mode, then don't leave it. */
837 RTCCUINTREG uHostCR4 = ASMGetCR4();
838
839 int rc;
840 if (uHostCR4 & X86_CR4_VMXE)
841 {
842 /* Exit VMX root mode and clear the VMX bit in CR4. */
843 VMXDisable();
844 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
845 rc = VINF_SUCCESS;
846 }
847 else
848 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
849
850 /* Restore interrupts. */
851 ASMSetFlags(fEFlags);
852 return rc;
853}
854
855
856/**
857 * Allocates and maps one physically contiguous page. The allocated page is
858 * zero'd out. (Used by various VT-x structures).
859 *
860 * @returns IPRT status code.
861 * @param pMemObj Pointer to the ring-0 memory object.
862 * @param ppVirt Where to store the virtual address of the
863 * allocation.
864 * @param pHCPhys Where to store the physical address of the
865 * allocation.
866 */
867static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
868{
869 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
872
873 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
874 if (RT_FAILURE(rc))
875 return rc;
876 *ppVirt = RTR0MemObjAddress(*pMemObj);
877 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
878 ASMMemZero32(*ppVirt, PAGE_SIZE);
879 return VINF_SUCCESS;
880}
881
882
883/**
884 * Frees and unmaps an allocated physical page.
885 *
886 * @param pMemObj Pointer to the ring-0 memory object.
887 * @param ppVirt Where to re-initialize the virtual address of
888 * allocation as 0.
889 * @param pHCPhys Where to re-initialize the physical address of the
890 * allocation as 0.
891 */
892static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
893{
894 AssertPtr(pMemObj);
895 AssertPtr(ppVirt);
896 AssertPtr(pHCPhys);
897 if (*pMemObj != NIL_RTR0MEMOBJ)
898 {
899 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
900 AssertRC(rc);
901 *pMemObj = NIL_RTR0MEMOBJ;
902 *ppVirt = 0;
903 *pHCPhys = 0;
904 }
905}
906
907
908/**
909 * Worker function to free VT-x related structures.
910 *
911 * @returns IPRT status code.
912 * @param pVM The cross context VM structure.
913 */
914static void hmR0VmxStructsFree(PVM pVM)
915{
916 for (VMCPUID i = 0; i < pVM->cCpus; i++)
917 {
918 PVMCPU pVCpu = &pVM->aCpus[i];
919 AssertPtr(pVCpu);
920
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
923
924 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
925 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
926
927 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
928 }
929
930 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
931#ifdef VBOX_WITH_CRASHDUMP_MAGIC
932 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
933#endif
934}
935
936
937/**
938 * Worker function to allocate VT-x related VM structures.
939 *
940 * @returns IPRT status code.
941 * @param pVM The cross context VM structure.
942 */
943static int hmR0VmxStructsAlloc(PVM pVM)
944{
945 /*
946 * Initialize members up-front so we can cleanup properly on allocation failure.
947 */
948#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
949 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
950 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
951 pVM->hm.s.vmx.HCPhys##a_Name = 0;
952
953#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
954 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
955 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
956 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
957
958#ifdef VBOX_WITH_CRASHDUMP_MAGIC
959 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
960#endif
961 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
962
963 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
964 for (VMCPUID i = 0; i < pVM->cCpus; i++)
965 {
966 PVMCPU pVCpu = &pVM->aCpus[i];
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
971 }
972#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
973#undef VMXLOCAL_INIT_VM_MEMOBJ
974
975 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
976 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
977 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
978 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
979
980 /*
981 * Allocate all the VT-x structures.
982 */
983 int rc = VINF_SUCCESS;
984#ifdef VBOX_WITH_CRASHDUMP_MAGIC
985 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
986 if (RT_FAILURE(rc))
987 goto cleanup;
988 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
989 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
990#endif
991
992 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
993 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
994 {
995 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
996 &pVM->hm.s.vmx.HCPhysApicAccess);
997 if (RT_FAILURE(rc))
998 goto cleanup;
999 }
1000
1001 /*
1002 * Initialize per-VCPU VT-x structures.
1003 */
1004 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1005 {
1006 PVMCPU pVCpu = &pVM->aCpus[i];
1007 AssertPtr(pVCpu);
1008
1009 /* Allocate the VM control structure (VMCS). */
1010 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1011 if (RT_FAILURE(rc))
1012 goto cleanup;
1013
1014 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1015 if ( PDMHasApic(pVM)
1016 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1017 {
1018 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1019 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1020 if (RT_FAILURE(rc))
1021 goto cleanup;
1022 }
1023
1024 /*
1025 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1026 * transparent accesses of specific MSRs.
1027 *
1028 * If the condition for enabling MSR bitmaps changes here, don't forget to
1029 * update HMAreMsrBitmapsAvailable().
1030 */
1031 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1032 {
1033 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1034 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1035 if (RT_FAILURE(rc))
1036 goto cleanup;
1037 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1038 }
1039
1040 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1041 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1042 if (RT_FAILURE(rc))
1043 goto cleanup;
1044
1045 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1046 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1047 if (RT_FAILURE(rc))
1048 goto cleanup;
1049 }
1050
1051 return VINF_SUCCESS;
1052
1053cleanup:
1054 hmR0VmxStructsFree(pVM);
1055 return rc;
1056}
1057
1058
1059/**
1060 * Does global VT-x initialization (called during module initialization).
1061 *
1062 * @returns VBox status code.
1063 */
1064VMMR0DECL(int) VMXR0GlobalInit(void)
1065{
1066#ifdef HMVMX_USE_FUNCTION_TABLE
1067 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1068# ifdef VBOX_STRICT
1069 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1070 Assert(g_apfnVMExitHandlers[i]);
1071# endif
1072#endif
1073 return VINF_SUCCESS;
1074}
1075
1076
1077/**
1078 * Does global VT-x termination (called during module termination).
1079 */
1080VMMR0DECL(void) VMXR0GlobalTerm()
1081{
1082 /* Nothing to do currently. */
1083}
1084
1085
1086/**
1087 * Sets up and activates VT-x on the current CPU.
1088 *
1089 * @returns VBox status code.
1090 * @param pHostCpu Pointer to the global CPU info struct.
1091 * @param pVM The cross context VM structure. Can be
1092 * NULL after a host resume operation.
1093 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1094 * fEnabledByHost is @c true).
1095 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1096 * @a fEnabledByHost is @c true).
1097 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1098 * enable VT-x on the host.
1099 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1100 */
1101VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1102 void *pvMsrs)
1103{
1104 Assert(pHostCpu);
1105 Assert(pvMsrs);
1106 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1107
1108 /* Enable VT-x if it's not already enabled by the host. */
1109 if (!fEnabledByHost)
1110 {
1111 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1112 if (RT_FAILURE(rc))
1113 return rc;
1114 }
1115
1116 /*
1117 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1118 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1119 * invalidated when flushing by VPID.
1120 */
1121 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1122 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1123 {
1124 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1125 pHostCpu->fFlushAsidBeforeUse = false;
1126 }
1127 else
1128 pHostCpu->fFlushAsidBeforeUse = true;
1129
1130 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1131 ++pHostCpu->cTlbFlushes;
1132
1133 return VINF_SUCCESS;
1134}
1135
1136
1137/**
1138 * Deactivates VT-x on the current CPU.
1139 *
1140 * @returns VBox status code.
1141 * @param pHostCpu Pointer to the global CPU info struct.
1142 * @param pvCpuPage Pointer to the VMXON region.
1143 * @param HCPhysCpuPage Physical address of the VMXON region.
1144 *
1145 * @remarks This function should never be called when SUPR0EnableVTx() or
1146 * similar was used to enable VT-x on the host.
1147 */
1148VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1149{
1150 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1151
1152 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1153 return hmR0VmxLeaveRootMode();
1154}
1155
1156
1157/**
1158 * Sets the permission bits for the specified MSR in the MSR bitmap.
1159 *
1160 * @param pVCpu The cross context virtual CPU structure.
1161 * @param uMsr The MSR value.
1162 * @param enmRead Whether reading this MSR causes a VM-exit.
1163 * @param enmWrite Whether writing this MSR causes a VM-exit.
1164 */
1165static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1166{
1167 int32_t iBit;
1168 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1169
1170 /*
1171 * Layout:
1172 * 0x000 - 0x3ff - Low MSR read bits
1173 * 0x400 - 0x7ff - High MSR read bits
1174 * 0x800 - 0xbff - Low MSR write bits
1175 * 0xc00 - 0xfff - High MSR write bits
1176 */
1177 if (uMsr <= 0x00001FFF)
1178 iBit = uMsr;
1179 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1180 {
1181 iBit = uMsr - UINT32_C(0xC0000000);
1182 pbMsrBitmap += 0x400;
1183 }
1184 else
1185 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1186
1187 Assert(iBit <= 0x1fff);
1188 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1189 ASMBitSet(pbMsrBitmap, iBit);
1190 else
1191 ASMBitClear(pbMsrBitmap, iBit);
1192
1193 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1194 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1195 else
1196 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1197}
1198
1199
1200#ifdef VBOX_STRICT
1201/**
1202 * Gets the permission bits for the specified MSR in the MSR bitmap.
1203 *
1204 * @returns VBox status code.
1205 * @retval VINF_SUCCESS if the specified MSR is found.
1206 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1207 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1208 *
1209 * @param pVCpu The cross context virtual CPU structure.
1210 * @param uMsr The MSR.
1211 * @param penmRead Where to store the read permissions.
1212 * @param penmWrite Where to store the write permissions.
1213 */
1214static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1215{
1216 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1217 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1218 int32_t iBit;
1219 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1220
1221 /* See hmR0VmxSetMsrPermission() for the layout. */
1222 if (uMsr <= 0x00001FFF)
1223 iBit = uMsr;
1224 else if ( uMsr >= 0xC0000000
1225 && uMsr <= 0xC0001FFF)
1226 {
1227 iBit = (uMsr - 0xC0000000);
1228 pbMsrBitmap += 0x400;
1229 }
1230 else
1231 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1232
1233 Assert(iBit <= 0x1fff);
1234 if (ASMBitTest(pbMsrBitmap, iBit))
1235 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1236 else
1237 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1238
1239 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1240 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1241 else
1242 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1243 return VINF_SUCCESS;
1244}
1245#endif /* VBOX_STRICT */
1246
1247
1248/**
1249 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1250 * area.
1251 *
1252 * @returns VBox status code.
1253 * @param pVCpu The cross context virtual CPU structure.
1254 * @param cMsrs The number of MSRs.
1255 */
1256static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1257{
1258 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1259 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1260 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1261 {
1262 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1263 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1264 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1265 }
1266
1267 /* Update number of guest MSRs to load/store across the world-switch. */
1268 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1269 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1270
1271 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1273 AssertRCReturn(rc, rc);
1274
1275 /* Update the VCPU's copy of the MSR count. */
1276 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1277
1278 return VINF_SUCCESS;
1279}
1280
1281
1282/**
1283 * Adds a new (or updates the value of an existing) guest/host MSR
1284 * pair to be swapped during the world-switch as part of the
1285 * auto-load/store MSR area in the VMCS.
1286 *
1287 * @returns VBox status code.
1288 * @param pVCpu The cross context virtual CPU structure.
1289 * @param uMsr The MSR.
1290 * @param uGuestMsrValue Value of the guest MSR.
1291 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1292 * necessary.
1293 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1294 * its value was updated. Optional, can be NULL.
1295 */
1296static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1297 bool *pfAddedAndUpdated)
1298{
1299 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1300 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1301 uint32_t i;
1302 for (i = 0; i < cMsrs; i++)
1303 {
1304 if (pGuestMsr->u32Msr == uMsr)
1305 break;
1306 pGuestMsr++;
1307 }
1308
1309 bool fAdded = false;
1310 if (i == cMsrs)
1311 {
1312 ++cMsrs;
1313 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1314 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1315
1316 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1317 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1318 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1319
1320 fAdded = true;
1321 }
1322
1323 /* Update the MSR values in the auto-load/store MSR area. */
1324 pGuestMsr->u32Msr = uMsr;
1325 pGuestMsr->u64Value = uGuestMsrValue;
1326
1327 /* Create/update the MSR slot in the host MSR area. */
1328 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1329 pHostMsr += i;
1330 pHostMsr->u32Msr = uMsr;
1331
1332 /*
1333 * Update the host MSR only when requested by the caller AND when we're
1334 * adding it to the auto-load/store area. Otherwise, it would have been
1335 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1336 */
1337 bool fUpdatedMsrValue = false;
1338 if ( fAdded
1339 && fUpdateHostMsr)
1340 {
1341 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1342 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1343 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1344 fUpdatedMsrValue = true;
1345 }
1346
1347 if (pfAddedAndUpdated)
1348 *pfAddedAndUpdated = fUpdatedMsrValue;
1349 return VINF_SUCCESS;
1350}
1351
1352
1353/**
1354 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1355 * auto-load/store MSR area in the VMCS.
1356 *
1357 * @returns VBox status code.
1358 * @param pVCpu The cross context virtual CPU structure.
1359 * @param uMsr The MSR.
1360 */
1361static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1362{
1363 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1364 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1365 for (uint32_t i = 0; i < cMsrs; i++)
1366 {
1367 /* Find the MSR. */
1368 if (pGuestMsr->u32Msr == uMsr)
1369 {
1370 /* If it's the last MSR, simply reduce the count. */
1371 if (i == cMsrs - 1)
1372 {
1373 --cMsrs;
1374 break;
1375 }
1376
1377 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1378 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1379 pLastGuestMsr += cMsrs - 1;
1380 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1381 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1382
1383 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1385 pLastHostMsr += cMsrs - 1;
1386 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1387 pHostMsr->u64Value = pLastHostMsr->u64Value;
1388 --cMsrs;
1389 break;
1390 }
1391 pGuestMsr++;
1392 }
1393
1394 /* Update the VMCS if the count changed (meaning the MSR was found). */
1395 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1396 {
1397 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1398 AssertRCReturn(rc, rc);
1399
1400 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1401 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1402 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1403
1404 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1405 return VINF_SUCCESS;
1406 }
1407
1408 return VERR_NOT_FOUND;
1409}
1410
1411
1412/**
1413 * Checks if the specified guest MSR is part of the auto-load/store area in
1414 * the VMCS.
1415 *
1416 * @returns true if found, false otherwise.
1417 * @param pVCpu The cross context virtual CPU structure.
1418 * @param uMsr The MSR to find.
1419 */
1420static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1421{
1422 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1423 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1424
1425 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1426 {
1427 if (pGuestMsr->u32Msr == uMsr)
1428 return true;
1429 }
1430 return false;
1431}
1432
1433
1434/**
1435 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1436 *
1437 * @param pVCpu The cross context virtual CPU structure.
1438 *
1439 * @remarks No-long-jump zone!!!
1440 */
1441static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1442{
1443 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1444 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1445 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1446 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1447
1448 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1449 {
1450 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1451
1452 /*
1453 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1454 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1455 */
1456 if (pHostMsr->u32Msr == MSR_K6_EFER)
1457 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1458 else
1459 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1460 }
1461
1462 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1463}
1464
1465
1466/**
1467 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1468 * perform lazy restoration of the host MSRs while leaving VT-x.
1469 *
1470 * @param pVCpu The cross context virtual CPU structure.
1471 *
1472 * @remarks No-long-jump zone!!!
1473 */
1474static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1475{
1476 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1477
1478 /*
1479 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1480 */
1481 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1482 {
1483 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1484#if HC_ARCH_BITS == 64
1485 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1486 {
1487 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1488 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1489 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1490 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1491 }
1492#endif
1493 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1494 }
1495}
1496
1497
1498/**
1499 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1500 * lazily while leaving VT-x.
1501 *
1502 * @returns true if it does, false otherwise.
1503 * @param pVCpu The cross context virtual CPU structure.
1504 * @param uMsr The MSR to check.
1505 */
1506static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1507{
1508 NOREF(pVCpu);
1509#if HC_ARCH_BITS == 64
1510 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1511 {
1512 switch (uMsr)
1513 {
1514 case MSR_K8_LSTAR:
1515 case MSR_K6_STAR:
1516 case MSR_K8_SF_MASK:
1517 case MSR_K8_KERNEL_GS_BASE:
1518 return true;
1519 }
1520 }
1521#else
1522 RT_NOREF(pVCpu, uMsr);
1523#endif
1524 return false;
1525}
1526
1527
1528/**
1529 * Loads a set of guests MSRs to allow read/passthru to the guest.
1530 *
1531 * The name of this function is slightly confusing. This function does NOT
1532 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1533 * common prefix for functions dealing with "lazy restoration" of the shared
1534 * MSRs.
1535 *
1536 * @param pVCpu The cross context virtual CPU structure.
1537 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1538 * out-of-sync. Make sure to update the required fields
1539 * before using them.
1540 *
1541 * @remarks No-long-jump zone!!!
1542 */
1543static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1544{
1545 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1546 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1547
1548 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1549#if HC_ARCH_BITS == 64
1550 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1551 {
1552 /*
1553 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1554 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1555 * we can skip a few MSR writes.
1556 *
1557 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1558 * guest MSR values in the guest-CPU context might be different to what's currently
1559 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1560 * CPU, see @bugref{8728}.
1561 */
1562 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1563 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1564 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1565 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1566 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1567 {
1568#ifdef VBOX_STRICT
1569 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1570 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1571 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1572 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1573#endif
1574 }
1575 else
1576 {
1577 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1578 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1579 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1580 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1581 }
1582 }
1583#else
1584 RT_NOREF(pMixedCtx);
1585#endif
1586 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1587}
1588
1589
1590/**
1591 * Performs lazy restoration of the set of host MSRs if they were previously
1592 * loaded with guest MSR values.
1593 *
1594 * @param pVCpu The cross context virtual CPU structure.
1595 *
1596 * @remarks No-long-jump zone!!!
1597 * @remarks The guest MSRs should have been saved back into the guest-CPU
1598 * context by hmR0VmxImportGuestState()!!!
1599 */
1600static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1601{
1602 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1603 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1604
1605 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1606 {
1607 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1608#if HC_ARCH_BITS == 64
1609 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1610 {
1611 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1612 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1613 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1614 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1615 }
1616#endif
1617 }
1618 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1619}
1620
1621
1622/**
1623 * Verifies that our cached values of the VMCS controls are all
1624 * consistent with what's actually present in the VMCS.
1625 *
1626 * @returns VBox status code.
1627 * @param pVCpu The cross context virtual CPU structure.
1628 */
1629static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1630{
1631 uint32_t u32Val;
1632 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1633 AssertRCReturn(rc, rc);
1634 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1635 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1636
1637 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1638 AssertRCReturn(rc, rc);
1639 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1640 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1641
1642 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1643 AssertRCReturn(rc, rc);
1644 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1645 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1646
1647 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1648 AssertRCReturn(rc, rc);
1649 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1650 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1651
1652 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1653 {
1654 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1655 AssertRCReturn(rc, rc);
1656 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1657 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1658 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1659 }
1660
1661 return VINF_SUCCESS;
1662}
1663
1664
1665#ifdef VBOX_STRICT
1666/**
1667 * Verifies that our cached host EFER value has not changed
1668 * since we cached it.
1669 *
1670 * @param pVCpu The cross context virtual CPU structure.
1671 */
1672static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1673{
1674 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1675
1676 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1677 {
1678 uint64_t u64Val;
1679 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1680 AssertRC(rc);
1681
1682 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1683 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1684 }
1685}
1686
1687
1688/**
1689 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1690 * VMCS are correct.
1691 *
1692 * @param pVCpu The cross context virtual CPU structure.
1693 */
1694static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1695{
1696 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1697
1698 /* Verify MSR counts in the VMCS are what we think it should be. */
1699 uint32_t cMsrs;
1700 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1701 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1702
1703 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1704 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1705
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1707 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1708
1709 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1710 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1711 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1712 {
1713 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1714 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1715 pGuestMsr->u32Msr, cMsrs));
1716
1717 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1718 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1719 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1720
1721 /* Verify that the permissions are as expected in the MSR bitmap. */
1722 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1723 {
1724 VMXMSREXITREAD enmRead;
1725 VMXMSREXITWRITE enmWrite;
1726 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1727 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1728 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1729 {
1730 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1731 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1732 }
1733 else
1734 {
1735 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1736 pGuestMsr->u32Msr, cMsrs));
1737 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1738 pGuestMsr->u32Msr, cMsrs));
1739 }
1740 }
1741 }
1742}
1743#endif /* VBOX_STRICT */
1744
1745
1746/**
1747 * Flushes the TLB using EPT.
1748 *
1749 * @returns VBox status code.
1750 * @param pVCpu The cross context virtual CPU structure of the calling
1751 * EMT. Can be NULL depending on @a enmFlush.
1752 * @param enmFlush Type of flush.
1753 *
1754 * @remarks Caller is responsible for making sure this function is called only
1755 * when NestedPaging is supported and providing @a enmFlush that is
1756 * supported by the CPU.
1757 * @remarks Can be called with interrupts disabled.
1758 */
1759static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1760{
1761 uint64_t au64Descriptor[2];
1762 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1763 au64Descriptor[0] = 0;
1764 else
1765 {
1766 Assert(pVCpu);
1767 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1768 }
1769 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1770
1771 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1772 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1773 rc));
1774 if ( RT_SUCCESS(rc)
1775 && pVCpu)
1776 {
1777 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1778 }
1779}
1780
1781
1782/**
1783 * Flushes the TLB using VPID.
1784 *
1785 * @returns VBox status code.
1786 * @param pVCpu The cross context virtual CPU structure of the calling
1787 * EMT. Can be NULL depending on @a enmFlush.
1788 * @param enmFlush Type of flush.
1789 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1790 * on @a enmFlush).
1791 *
1792 * @remarks Can be called with interrupts disabled.
1793 */
1794static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1795{
1796 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1797
1798 uint64_t au64Descriptor[2];
1799 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1800 {
1801 au64Descriptor[0] = 0;
1802 au64Descriptor[1] = 0;
1803 }
1804 else
1805 {
1806 AssertPtr(pVCpu);
1807 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1808 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1809 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1810 au64Descriptor[1] = GCPtr;
1811 }
1812
1813 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1814 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1815 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1816 if ( RT_SUCCESS(rc)
1817 && pVCpu)
1818 {
1819 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1820 }
1821 NOREF(rc);
1822}
1823
1824
1825/**
1826 * Invalidates a guest page by guest virtual address. Only relevant for
1827 * EPT/VPID, otherwise there is nothing really to invalidate.
1828 *
1829 * @returns VBox status code.
1830 * @param pVCpu The cross context virtual CPU structure.
1831 * @param GCVirt Guest virtual address of the page to invalidate.
1832 */
1833VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1834{
1835 AssertPtr(pVCpu);
1836 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1837
1838 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1839 if (!fFlushPending)
1840 {
1841 /*
1842 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1843 * the EPT case. See @bugref{6043} and @bugref{6177}.
1844 *
1845 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1846 * as this function maybe called in a loop with individual addresses.
1847 */
1848 PVM pVM = pVCpu->CTX_SUFF(pVM);
1849 if (pVM->hm.s.vmx.fVpid)
1850 {
1851 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1852
1853#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1854 /*
1855 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1856 * where executing INVVPID outside 64-bit mode does not flush translations of
1857 * 64-bit linear addresses, see @bugref{6208#c72}.
1858 */
1859 if (RT_HI_U32(GCVirt))
1860 fVpidFlush = false;
1861#endif
1862
1863 if (fVpidFlush)
1864 {
1865 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1866 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1867 }
1868 else
1869 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1870 }
1871 else if (pVM->hm.s.fNestedPaging)
1872 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1873 }
1874
1875 return VINF_SUCCESS;
1876}
1877
1878
1879/**
1880 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1881 * case where neither EPT nor VPID is supported by the CPU.
1882 *
1883 * @param pVCpu The cross context virtual CPU structure.
1884 * @param pCpu Pointer to the global HM struct.
1885 *
1886 * @remarks Called with interrupts disabled.
1887 */
1888static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1889{
1890 AssertPtr(pVCpu);
1891 AssertPtr(pCpu);
1892
1893 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1894
1895 Assert(pCpu->idCpu != NIL_RTCPUID);
1896 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1897 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1898 pVCpu->hm.s.fForceTLBFlush = false;
1899 return;
1900}
1901
1902
1903/**
1904 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1905 *
1906 * @param pVCpu The cross context virtual CPU structure.
1907 * @param pCpu Pointer to the global HM CPU struct.
1908 *
1909 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1910 * nomenclature. The reason is, to avoid confusion in compare statements
1911 * since the host-CPU copies are named "ASID".
1912 *
1913 * @remarks Called with interrupts disabled.
1914 */
1915static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1916{
1917#ifdef VBOX_WITH_STATISTICS
1918 bool fTlbFlushed = false;
1919# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1920# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1921 if (!fTlbFlushed) \
1922 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1923 } while (0)
1924#else
1925# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1926# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1927#endif
1928
1929 AssertPtr(pCpu);
1930 AssertPtr(pVCpu);
1931 Assert(pCpu->idCpu != NIL_RTCPUID);
1932
1933 PVM pVM = pVCpu->CTX_SUFF(pVM);
1934 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1935 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1936 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1937
1938 /*
1939 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1940 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1941 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1942 * cannot reuse the current ASID anymore.
1943 */
1944 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1945 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1946 {
1947 ++pCpu->uCurrentAsid;
1948 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1949 {
1950 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1951 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1952 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1953 }
1954
1955 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1956 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1957 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1958
1959 /*
1960 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1961 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1962 */
1963 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1964 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1965 HMVMX_SET_TAGGED_TLB_FLUSHED();
1966 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1967 }
1968
1969 /* Check for explicit TLB flushes. */
1970 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1971 {
1972 /*
1973 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1974 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1975 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1976 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1977 * mappings, see @bugref{6568}.
1978 *
1979 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1980 */
1981 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1982 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1983 HMVMX_SET_TAGGED_TLB_FLUSHED();
1984 }
1985
1986 pVCpu->hm.s.fForceTLBFlush = false;
1987 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1988
1989 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1990 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1991 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1992 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1993 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1994 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1995 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1996 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1997 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1998
1999 /* Update VMCS with the VPID. */
2000 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2001 AssertRC(rc);
2002
2003#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2004}
2005
2006
2007/**
2008 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2009 *
2010 * @returns VBox status code.
2011 * @param pVCpu The cross context virtual CPU structure.
2012 * @param pCpu Pointer to the global HM CPU struct.
2013 *
2014 * @remarks Called with interrupts disabled.
2015 */
2016static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2017{
2018 AssertPtr(pVCpu);
2019 AssertPtr(pCpu);
2020 Assert(pCpu->idCpu != NIL_RTCPUID);
2021 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2022 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2023
2024 /*
2025 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2026 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2027 */
2028 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2029 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2030 {
2031 pVCpu->hm.s.fForceTLBFlush = true;
2032 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2033 }
2034
2035 /* Check for explicit TLB flushes. */
2036 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2037 {
2038 pVCpu->hm.s.fForceTLBFlush = true;
2039 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2040 }
2041
2042 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2043 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2044
2045 if (pVCpu->hm.s.fForceTLBFlush)
2046 {
2047 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2048 pVCpu->hm.s.fForceTLBFlush = false;
2049 }
2050}
2051
2052
2053/**
2054 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2055 *
2056 * @returns VBox status code.
2057 * @param pVCpu The cross context virtual CPU structure.
2058 * @param pCpu Pointer to the global HM CPU struct.
2059 *
2060 * @remarks Called with interrupts disabled.
2061 */
2062static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2063{
2064 AssertPtr(pVCpu);
2065 AssertPtr(pCpu);
2066 Assert(pCpu->idCpu != NIL_RTCPUID);
2067 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2068 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2069
2070 /*
2071 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2072 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2073 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2074 * cannot reuse the current ASID anymore.
2075 */
2076 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2077 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2078 {
2079 pVCpu->hm.s.fForceTLBFlush = true;
2080 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2081 }
2082
2083 /* Check for explicit TLB flushes. */
2084 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2085 {
2086 /*
2087 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2088 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2089 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2090 * include fExplicitFlush's too) - an obscure corner case.
2091 */
2092 pVCpu->hm.s.fForceTLBFlush = true;
2093 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2094 }
2095
2096 PVM pVM = pVCpu->CTX_SUFF(pVM);
2097 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2098 if (pVCpu->hm.s.fForceTLBFlush)
2099 {
2100 ++pCpu->uCurrentAsid;
2101 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2102 {
2103 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2104 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2105 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2106 }
2107
2108 pVCpu->hm.s.fForceTLBFlush = false;
2109 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2110 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2111 if (pCpu->fFlushAsidBeforeUse)
2112 {
2113 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2114 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2115 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2116 {
2117 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2118 pCpu->fFlushAsidBeforeUse = false;
2119 }
2120 else
2121 {
2122 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2123 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2124 }
2125 }
2126 }
2127
2128 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2129 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2130 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2131 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2132 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2133 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2134 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2135
2136 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2137 AssertRC(rc);
2138}
2139
2140
2141/**
2142 * Flushes the guest TLB entry based on CPU capabilities.
2143 *
2144 * @param pVCpu The cross context virtual CPU structure.
2145 * @param pCpu Pointer to the global HM CPU struct.
2146 */
2147DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2148{
2149#ifdef HMVMX_ALWAYS_FLUSH_TLB
2150 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2151#endif
2152 PVM pVM = pVCpu->CTX_SUFF(pVM);
2153 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2154 {
2155 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2156 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2158 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2159 default:
2160 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2161 break;
2162 }
2163 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2164}
2165
2166
2167/**
2168 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2169 * TLB entries from the host TLB before VM-entry.
2170 *
2171 * @returns VBox status code.
2172 * @param pVM The cross context VM structure.
2173 */
2174static int hmR0VmxSetupTaggedTlb(PVM pVM)
2175{
2176 /*
2177 * Determine optimal flush type for Nested Paging.
2178 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2179 * guest execution (see hmR3InitFinalizeR0()).
2180 */
2181 if (pVM->hm.s.fNestedPaging)
2182 {
2183 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2184 {
2185 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2186 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2187 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2188 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2189 else
2190 {
2191 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2192 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2193 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2194 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2195 }
2196
2197 /* Make sure the write-back cacheable memory type for EPT is supported. */
2198 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2199 {
2200 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2201 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2202 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2203 }
2204
2205 /* EPT requires a page-walk length of 4. */
2206 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2207 {
2208 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2209 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2210 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2211 }
2212 }
2213 else
2214 {
2215 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2216 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2217 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2218 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2219 }
2220 }
2221
2222 /*
2223 * Determine optimal flush type for VPID.
2224 */
2225 if (pVM->hm.s.vmx.fVpid)
2226 {
2227 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2228 {
2229 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2230 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2231 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2232 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2233 else
2234 {
2235 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2236 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2237 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2238 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2239 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2240 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2241 pVM->hm.s.vmx.fVpid = false;
2242 }
2243 }
2244 else
2245 {
2246 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2247 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2248 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2249 pVM->hm.s.vmx.fVpid = false;
2250 }
2251 }
2252
2253 /*
2254 * Setup the handler for flushing tagged-TLBs.
2255 */
2256 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2257 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2258 else if (pVM->hm.s.fNestedPaging)
2259 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2260 else if (pVM->hm.s.vmx.fVpid)
2261 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2262 else
2263 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2264 return VINF_SUCCESS;
2265}
2266
2267
2268/**
2269 * Sets up pin-based VM-execution controls in the VMCS.
2270 *
2271 * @returns VBox status code.
2272 * @param pVCpu The cross context virtual CPU structure.
2273 *
2274 * @remarks We don't really care about optimizing vmwrites here as it's done only
2275 * once per VM and hence we don't care about VMCS-field cache comparisons.
2276 */
2277static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2278{
2279 PVM pVM = pVCpu->CTX_SUFF(pVM);
2280 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2281 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2282
2283 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2284 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2285
2286 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2287 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2288
2289 /* Enable the VMX preemption timer. */
2290 if (pVM->hm.s.vmx.fUsePreemptTimer)
2291 {
2292 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2293 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2294 }
2295
2296#if 0
2297 /* Enable posted-interrupt processing. */
2298 if (pVM->hm.s.fPostedIntrs)
2299 {
2300 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2301 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2302 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2303 }
2304#endif
2305
2306 if ((fVal & fZap) != fVal)
2307 {
2308 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2309 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2310 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2311 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2312 }
2313
2314 /* Commit it to the VMCS and update our cache. */
2315 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2316 AssertRCReturn(rc, rc);
2317 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2318
2319 return VINF_SUCCESS;
2320}
2321
2322
2323/**
2324 * Sets up secondary processor-based VM-execution controls in the VMCS.
2325 *
2326 * @returns VBox status code.
2327 * @param pVCpu The cross context virtual CPU structure.
2328 *
2329 * @remarks We don't really care about optimizing vmwrites here as it's done only
2330 * once per VM and hence we don't care about VMCS-field cache comparisons.
2331 */
2332static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2333{
2334 PVM pVM = pVCpu->CTX_SUFF(pVM);
2335 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2336 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2337
2338 /* WBINVD causes a VM-exit. */
2339 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2340 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2341
2342 /* Enable EPT (aka nested-paging). */
2343 if (pVM->hm.s.fNestedPaging)
2344 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2345
2346 /*
2347 * Enable the INVPCID instruction if supported by the hardware and we expose
2348 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2349 */
2350 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2351 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2352 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2353
2354 /* Enable VPID. */
2355 if (pVM->hm.s.vmx.fVpid)
2356 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2357
2358 /* Enable Unrestricted guest execution. */
2359 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2360 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2361
2362#if 0
2363 if (pVM->hm.s.fVirtApicRegs)
2364 {
2365 /* Enable APIC-register virtualization. */
2366 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2367 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2368
2369 /* Enable virtual-interrupt delivery. */
2370 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2371 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2372 }
2373#endif
2374
2375 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2376 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2377 * done dynamically. */
2378 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2379 {
2380 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2381 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2382 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2383 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2384 AssertRCReturn(rc, rc);
2385 }
2386
2387 /* Enable RDTSCP. */
2388 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2389 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2390
2391 /* Enable Pause-Loop exiting. */
2392 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2393 && pVM->hm.s.vmx.cPleGapTicks
2394 && pVM->hm.s.vmx.cPleWindowTicks)
2395 {
2396 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2397
2398 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2399 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2400 AssertRCReturn(rc, rc);
2401 }
2402
2403 if ((fVal & fZap) != fVal)
2404 {
2405 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2406 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2407 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2408 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2409 }
2410
2411 /* Commit it to the VMCS and update our cache. */
2412 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2413 AssertRCReturn(rc, rc);
2414 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2415
2416 return VINF_SUCCESS;
2417}
2418
2419
2420/**
2421 * Sets up processor-based VM-execution controls in the VMCS.
2422 *
2423 * @returns VBox status code.
2424 * @param pVCpu The cross context virtual CPU structure.
2425 *
2426 * @remarks We don't really care about optimizing vmwrites here as it's done only
2427 * once per VM and hence we don't care about VMCS-field cache comparisons.
2428 */
2429static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2430{
2431 PVM pVM = pVCpu->CTX_SUFF(pVM);
2432 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2433 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2434
2435 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2436 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2437 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2438 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2441 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2442
2443 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2444 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2445 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2446 {
2447 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2448 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2449 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2450 }
2451
2452 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2453 if (!pVM->hm.s.fNestedPaging)
2454 {
2455 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2456 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2458 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2459 }
2460
2461 /* Use TPR shadowing if supported by the CPU. */
2462 if ( PDMHasApic(pVM)
2463 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2464 {
2465 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2466 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2467 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2468 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2469 AssertRCReturn(rc, rc);
2470
2471 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2472 /* CR8 writes cause a VM-exit based on TPR threshold. */
2473 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2474 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2475 }
2476 else
2477 {
2478 /*
2479 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2480 * Set this control only for 64-bit guests.
2481 */
2482 if (pVM->hm.s.fAllow64BitGuests)
2483 {
2484 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2485 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2486 }
2487 }
2488
2489 /* Use MSR-bitmaps if supported by the CPU. */
2490 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2491 {
2492 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2493
2494 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2495 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2496 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2497 AssertRCReturn(rc, rc);
2498
2499 /*
2500 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2501 * automatically using dedicated fields in the VMCS.
2502 */
2503 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508#if HC_ARCH_BITS == 64
2509 /*
2510 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2511 */
2512 if (pVM->hm.s.fAllow64BitGuests)
2513 {
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 }
2519#endif
2520 /*
2521 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2522 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2523 */
2524 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2525 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2526
2527 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2528 }
2529
2530 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2531 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2532 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2533
2534 if ((fVal & fZap) != fVal)
2535 {
2536 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2537 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2538 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2539 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2540 }
2541
2542 /* Commit it to the VMCS and update our cache. */
2543 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2544 AssertRCReturn(rc, rc);
2545 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2546
2547 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2548 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2549 return hmR0VmxSetupProcCtls2(pVCpu);
2550
2551 /* Sanity check, should not really happen. */
2552 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2553 {
2554 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2555 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2556 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2557 }
2558
2559 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2560 return VINF_SUCCESS;
2561}
2562
2563
2564/**
2565 * Sets up miscellaneous (everything other than Pin & Processor-based
2566 * VM-execution) control fields in the VMCS.
2567 *
2568 * @returns VBox status code.
2569 * @param pVCpu The cross context virtual CPU structure.
2570 */
2571static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2572{
2573 AssertPtr(pVCpu);
2574
2575 int rc = VERR_GENERAL_FAILURE;
2576
2577 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2578#if 0
2579 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2580 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2581 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2582
2583 /*
2584 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2585 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2586 * We thus use the exception bitmap to control it rather than use both.
2587 */
2588 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2589 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2590
2591 /* All IO & IOIO instructions cause VM-exits. */
2592 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2593 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2594
2595 /* Initialize the MSR-bitmap area. */
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2599 AssertRCReturn(rc, rc);
2600#endif
2601
2602 /* Setup MSR auto-load/store area. */
2603 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2604 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2605 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2606 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607 AssertRCReturn(rc, rc);
2608
2609 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2610 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2611 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2612 AssertRCReturn(rc, rc);
2613
2614 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2615 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2616 AssertRCReturn(rc, rc);
2617
2618 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2619#if 0
2620 /* Setup debug controls */
2621 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2622 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2623 AssertRCReturn(rc, rc);
2624#endif
2625
2626 return rc;
2627}
2628
2629
2630/**
2631 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2632 *
2633 * We shall setup those exception intercepts that don't change during the
2634 * lifetime of the VM here. The rest are done dynamically while loading the
2635 * guest state.
2636 *
2637 * @returns VBox status code.
2638 * @param pVCpu The cross context virtual CPU structure.
2639 */
2640static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2641{
2642 AssertPtr(pVCpu);
2643
2644 uint32_t u32XcptBitmap;
2645
2646 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2647 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2648
2649 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2650 and writes, and because recursive #DBs can cause the CPU hang, we must always
2651 intercept #DB. */
2652 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2653
2654 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2655 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2656 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2657
2658 /* Commit it to the VMCS. */
2659 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2660 AssertRCReturn(rc, rc);
2661
2662 /* Update our cache of the exception bitmap. */
2663 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2664 return VINF_SUCCESS;
2665}
2666
2667
2668/**
2669 * Does per-VM VT-x initialization.
2670 *
2671 * @returns VBox status code.
2672 * @param pVM The cross context VM structure.
2673 */
2674VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2675{
2676 LogFlowFunc(("pVM=%p\n", pVM));
2677
2678 int rc = hmR0VmxStructsAlloc(pVM);
2679 if (RT_FAILURE(rc))
2680 {
2681 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2682 return rc;
2683 }
2684
2685 return VINF_SUCCESS;
2686}
2687
2688
2689/**
2690 * Does per-VM VT-x termination.
2691 *
2692 * @returns VBox status code.
2693 * @param pVM The cross context VM structure.
2694 */
2695VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2696{
2697 LogFlowFunc(("pVM=%p\n", pVM));
2698
2699#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2700 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2701 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2702#endif
2703 hmR0VmxStructsFree(pVM);
2704 return VINF_SUCCESS;
2705}
2706
2707
2708/**
2709 * Sets up the VM for execution under VT-x.
2710 * This function is only called once per-VM during initialization.
2711 *
2712 * @returns VBox status code.
2713 * @param pVM The cross context VM structure.
2714 */
2715VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2716{
2717 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2718 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2719
2720 LogFlowFunc(("pVM=%p\n", pVM));
2721
2722 /*
2723 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2724 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2725 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2726 */
2727 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2728 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2729 || !pVM->hm.s.vmx.pRealModeTSS))
2730 {
2731 LogRelFunc(("Invalid real-on-v86 state.\n"));
2732 return VERR_INTERNAL_ERROR;
2733 }
2734
2735 /* Initialize these always, see hmR3InitFinalizeR0().*/
2736 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2737 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2738
2739 /* Setup the tagged-TLB flush handlers. */
2740 int rc = hmR0VmxSetupTaggedTlb(pVM);
2741 if (RT_FAILURE(rc))
2742 {
2743 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2744 return rc;
2745 }
2746
2747 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2748 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2749#if HC_ARCH_BITS == 64
2750 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2751 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2752 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2753 {
2754 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2755 }
2756#endif
2757
2758 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2759 RTCCUINTREG uHostCR4 = ASMGetCR4();
2760 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2761 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2762
2763 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2764 {
2765 PVMCPU pVCpu = &pVM->aCpus[i];
2766 AssertPtr(pVCpu);
2767 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2768
2769 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2770 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2771
2772 /* Set revision dword at the beginning of the VMCS structure. */
2773 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2774
2775 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2776 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2777 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2778 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2779
2780 /* Load this VMCS as the current VMCS. */
2781 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2782 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2783 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2784
2785 rc = hmR0VmxSetupPinCtls(pVCpu);
2786 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2787 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2788
2789 rc = hmR0VmxSetupProcCtls(pVCpu);
2790 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2791 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2792
2793 rc = hmR0VmxSetupMiscCtls(pVCpu);
2794 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2795 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2796
2797 rc = hmR0VmxInitXcptBitmap(pVCpu);
2798 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2799 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2800
2801#if HC_ARCH_BITS == 32
2802 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2803 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2804 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2805#endif
2806
2807 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2808 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2809 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2810 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2811
2812 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2813
2814 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2815 }
2816
2817 return VINF_SUCCESS;
2818}
2819
2820
2821/**
2822 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2823 * the VMCS.
2824 *
2825 * @returns VBox status code.
2826 */
2827static int hmR0VmxExportHostControlRegs(void)
2828{
2829 RTCCUINTREG uReg = ASMGetCR0();
2830 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2831 AssertRCReturn(rc, rc);
2832
2833 uReg = ASMGetCR3();
2834 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2835 AssertRCReturn(rc, rc);
2836
2837 uReg = ASMGetCR4();
2838 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2839 AssertRCReturn(rc, rc);
2840 return rc;
2841}
2842
2843
2844/**
2845 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2846 * the host-state area in the VMCS.
2847 *
2848 * @returns VBox status code.
2849 * @param pVCpu The cross context virtual CPU structure.
2850 */
2851static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2852{
2853#if HC_ARCH_BITS == 64
2854/**
2855 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2856 * requirements. See hmR0VmxExportHostSegmentRegs().
2857 */
2858# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2859 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2860 { \
2861 bool fValidSelector = true; \
2862 if ((selValue) & X86_SEL_LDT) \
2863 { \
2864 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2865 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2866 } \
2867 if (fValidSelector) \
2868 { \
2869 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2870 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2871 } \
2872 (selValue) = 0; \
2873 }
2874
2875 /*
2876 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2877 * should -not- save the messed up state without restoring the original host-state,
2878 * see @bugref{7240}.
2879 *
2880 * This apparently can happen (most likely the FPU changes), deal with it rather than
2881 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2882 */
2883 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2884 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2885 {
2886 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2887 pVCpu->idCpu));
2888 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2889 }
2890 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2891#else
2892 RT_NOREF(pVCpu);
2893#endif
2894
2895 /*
2896 * Host DS, ES, FS and GS segment registers.
2897 */
2898#if HC_ARCH_BITS == 64
2899 RTSEL uSelDS = ASMGetDS();
2900 RTSEL uSelES = ASMGetES();
2901 RTSEL uSelFS = ASMGetFS();
2902 RTSEL uSelGS = ASMGetGS();
2903#else
2904 RTSEL uSelDS = 0;
2905 RTSEL uSelES = 0;
2906 RTSEL uSelFS = 0;
2907 RTSEL uSelGS = 0;
2908#endif
2909
2910 /*
2911 * Host CS and SS segment registers.
2912 */
2913 RTSEL uSelCS = ASMGetCS();
2914 RTSEL uSelSS = ASMGetSS();
2915
2916 /*
2917 * Host TR segment register.
2918 */
2919 RTSEL uSelTR = ASMGetTR();
2920
2921#if HC_ARCH_BITS == 64
2922 /*
2923 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2924 * gain VM-entry and restore them before we get preempted.
2925 *
2926 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2927 */
2928 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2929 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2930 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2931 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2932# undef VMXLOCAL_ADJUST_HOST_SEG
2933#endif
2934
2935 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2936 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2937 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2938 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2939 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2940 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2941 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2942 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2943 Assert(uSelCS);
2944 Assert(uSelTR);
2945
2946 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2947#if 0
2948 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2949 Assert(uSelSS != 0);
2950#endif
2951
2952 /* Write these host selector fields into the host-state area in the VMCS. */
2953 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2954 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2955#if HC_ARCH_BITS == 64
2956 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2959 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2960#else
2961 NOREF(uSelDS);
2962 NOREF(uSelES);
2963 NOREF(uSelFS);
2964 NOREF(uSelGS);
2965#endif
2966 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2967 AssertRCReturn(rc, rc);
2968
2969 /*
2970 * Host GDTR and IDTR.
2971 */
2972 RTGDTR Gdtr;
2973 RTIDTR Idtr;
2974 RT_ZERO(Gdtr);
2975 RT_ZERO(Idtr);
2976 ASMGetGDTR(&Gdtr);
2977 ASMGetIDTR(&Idtr);
2978 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2979 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2980 AssertRCReturn(rc, rc);
2981
2982#if HC_ARCH_BITS == 64
2983 /*
2984 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2985 * them to the maximum limit (0xffff) on every VM-exit.
2986 */
2987 if (Gdtr.cbGdt != 0xffff)
2988 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2989
2990 /*
2991 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2992 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2993 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2994 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2995 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2996 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2997 * at 0xffff on hosts where we are sure it won't cause trouble.
2998 */
2999# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3000 if (Idtr.cbIdt < 0x0fff)
3001# else
3002 if (Idtr.cbIdt != 0xffff)
3003# endif
3004 {
3005 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3006 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3007 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3008 }
3009#endif
3010
3011 /*
3012 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3013 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3014 * RPL should be too in most cases.
3015 */
3016 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3017 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3018
3019 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3020#if HC_ARCH_BITS == 64
3021 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3022
3023 /*
3024 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3025 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3026 * restoration if the host has something else. Task switching is not supported in 64-bit
3027 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3028 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3029 *
3030 * [1] See Intel spec. 3.5 "System Descriptor Types".
3031 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3032 */
3033 PVM pVM = pVCpu->CTX_SUFF(pVM);
3034 Assert(pDesc->System.u4Type == 11);
3035 if ( pDesc->System.u16LimitLow != 0x67
3036 || pDesc->System.u4LimitHigh)
3037 {
3038 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3039 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3040 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3041 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3042 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3043 }
3044
3045 /*
3046 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3047 */
3048 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3049 {
3050 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3051 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3052 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3053 {
3054 /* The GDT is read-only but the writable GDT is available. */
3055 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3056 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3057 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3058 AssertRCReturn(rc, rc);
3059 }
3060 }
3061#else
3062 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3063#endif
3064 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3065 AssertRCReturn(rc, rc);
3066
3067 /*
3068 * Host FS base and GS base.
3069 */
3070#if HC_ARCH_BITS == 64
3071 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3072 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3073 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3074 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3075 AssertRCReturn(rc, rc);
3076
3077 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3078 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3079 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3080 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3081 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3082#endif
3083 return VINF_SUCCESS;
3084}
3085
3086
3087/**
3088 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3089 * host-state area of the VMCS.
3090 *
3091 * Theses MSRs will be automatically restored on the host after every successful
3092 * VM-exit.
3093 *
3094 * @returns VBox status code.
3095 * @param pVCpu The cross context virtual CPU structure.
3096 *
3097 * @remarks No-long-jump zone!!!
3098 */
3099static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3100{
3101 AssertPtr(pVCpu);
3102 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3103
3104 /*
3105 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3106 * rather than swapping them on every VM-entry.
3107 */
3108 hmR0VmxLazySaveHostMsrs(pVCpu);
3109
3110 /*
3111 * Host Sysenter MSRs.
3112 */
3113 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3114#if HC_ARCH_BITS == 32
3115 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3116 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3117#else
3118 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3119 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3120#endif
3121 AssertRCReturn(rc, rc);
3122
3123 /*
3124 * Host EFER MSR.
3125 *
3126 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3127 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3128 */
3129 PVM pVM = pVCpu->CTX_SUFF(pVM);
3130 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3131 {
3132 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3133 AssertRCReturn(rc, rc);
3134 }
3135
3136 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3137
3138 return VINF_SUCCESS;
3139}
3140
3141
3142/**
3143 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3144 *
3145 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3146 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3147 * hmR0VMxExportGuestEntryCtls().
3148 *
3149 * @returns true if we need to load guest EFER, false otherwise.
3150 * @param pVCpu The cross context virtual CPU structure.
3151 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3152 * out-of-sync. Make sure to update the required fields
3153 * before using them.
3154 *
3155 * @remarks Requires EFER, CR4.
3156 * @remarks No-long-jump zone!!!
3157 */
3158static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3159{
3160#ifdef HMVMX_ALWAYS_SWAP_EFER
3161 return true;
3162#endif
3163
3164#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3165 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3166 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3167 return false;
3168#endif
3169
3170 PVM pVM = pVCpu->CTX_SUFF(pVM);
3171 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3172 uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3173
3174 /*
3175 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3176 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3177 */
3178 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3179 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3180 {
3181 return true;
3182 }
3183
3184 /*
3185 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3186 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3187 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3188 */
3189 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3190 && (pMixedCtx->cr0 & X86_CR0_PG)
3191 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3192 {
3193 /* Assert that host is PAE capable. */
3194 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3195 return true;
3196 }
3197
3198 return false;
3199}
3200
3201
3202/**
3203 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3204 *
3205 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3206 * see Intel spec. 24.8.1 "VM-entry controls".
3207 *
3208 * @returns VBox status code.
3209 * @param pVCpu The cross context virtual CPU structure.
3210 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3211 * out-of-sync. Make sure to update the required fields
3212 * before using them.
3213 *
3214 * @remarks Requires EFER.
3215 * @remarks No-long-jump zone!!!
3216 */
3217static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3218{
3219 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3220 {
3221 PVM pVM = pVCpu->CTX_SUFF(pVM);
3222 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3223 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3224
3225 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3226 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3227
3228 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3229 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3230 {
3231 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3232 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3233 }
3234 else
3235 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3236
3237 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3238 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3239 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3240 {
3241 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3242 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3243 }
3244
3245 /*
3246 * The following should -not- be set (since we're not in SMM mode):
3247 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3248 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3249 */
3250
3251 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3252 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3253
3254 if ((fVal & fZap) != fVal)
3255 {
3256 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3257 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3258 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3259 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3260 }
3261
3262 /* Commit it to the VMCS and update our cache. */
3263 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3264 {
3265 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3266 AssertRCReturn(rc, rc);
3267 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3268 }
3269
3270 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3271 }
3272 return VINF_SUCCESS;
3273}
3274
3275
3276/**
3277 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3278 *
3279 * @returns VBox status code.
3280 * @param pVCpu The cross context virtual CPU structure.
3281 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3282 * out-of-sync. Make sure to update the required fields
3283 * before using them.
3284 *
3285 * @remarks Requires EFER.
3286 */
3287static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3288{
3289 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3290 {
3291 PVM pVM = pVCpu->CTX_SUFF(pVM);
3292 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3293 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3294
3295 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3296 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3297
3298 /*
3299 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3300 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3301 * hmR0VmxExportHostMsrs().
3302 */
3303#if HC_ARCH_BITS == 64
3304 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3305 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3306#else
3307 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3308 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3309 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3310 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3311 {
3312 /* The switcher returns to long mode, EFER is managed by the switcher. */
3313 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3314 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3315 }
3316 else
3317 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3318#endif
3319
3320 /* If the newer VMCS fields for managing EFER exists, use it. */
3321 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3322 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3323 {
3324 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3325 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3326 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3327 }
3328
3329 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3330 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3331
3332 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3333 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3334 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3335
3336 /* Enable saving of the VMX preemption timer value on VM-exit. */
3337 if ( pVM->hm.s.vmx.fUsePreemptTimer
3338 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3339 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3340
3341 if ((fVal & fZap) != fVal)
3342 {
3343 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3344 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3345 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3346 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3347 }
3348
3349 /* Commit it to the VMCS and update our cache. */
3350 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3351 {
3352 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3353 AssertRCReturn(rc, rc);
3354 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3355 }
3356
3357 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3358 }
3359 return VINF_SUCCESS;
3360}
3361
3362
3363/**
3364 * Sets the TPR threshold in the VMCS.
3365 *
3366 * @returns VBox status code.
3367 * @param pVCpu The cross context virtual CPU structure.
3368 * @param u32TprThreshold The TPR threshold (task-priority class only).
3369 */
3370DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3371{
3372 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3373 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3374 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3375}
3376
3377
3378/**
3379 * Exports the guest APIC TPR state into the VMCS.
3380 *
3381 * @returns VBox status code.
3382 * @param pVCpu The cross context virtual CPU structure.
3383 *
3384 * @remarks No-long-jump zone!!!
3385 */
3386static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3387{
3388 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3389 {
3390 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3391
3392 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3393 && APICIsEnabled(pVCpu))
3394 {
3395 /*
3396 * Setup TPR shadowing.
3397 */
3398 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3399 {
3400 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3401
3402 bool fPendingIntr = false;
3403 uint8_t u8Tpr = 0;
3404 uint8_t u8PendingIntr = 0;
3405 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3406 AssertRCReturn(rc, rc);
3407
3408 /*
3409 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3410 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3411 * priority of the pending interrupt so we can deliver the interrupt. If there
3412 * are no interrupts pending, set threshold to 0 to not cause any
3413 * TPR-below-threshold VM-exits.
3414 */
3415 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3416 uint32_t u32TprThreshold = 0;
3417 if (fPendingIntr)
3418 {
3419 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3420 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3421 const uint8_t u8TprPriority = u8Tpr >> 4;
3422 if (u8PendingPriority <= u8TprPriority)
3423 u32TprThreshold = u8PendingPriority;
3424 }
3425
3426 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3427 AssertRCReturn(rc, rc);
3428 }
3429 }
3430 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3431 }
3432 return VINF_SUCCESS;
3433}
3434
3435
3436/**
3437 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3438 *
3439 * @returns Guest's interruptibility-state.
3440 * @param pVCpu The cross context virtual CPU structure.
3441 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3442 * out-of-sync. Make sure to update the required fields
3443 * before using them.
3444 *
3445 * @remarks No-long-jump zone!!!
3446 */
3447static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3448{
3449 /*
3450 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3451 */
3452 uint32_t fIntrState = 0;
3453 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3454 {
3455 /* If inhibition is active, RIP & RFLAGS should've been accessed
3456 (i.e. read previously from the VMCS or from ring-3). */
3457#ifdef VBOX_STRICT
3458 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3459 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3460#endif
3461 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3462 {
3463 if (pMixedCtx->eflags.Bits.u1IF)
3464 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3465 else
3466 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3467 }
3468 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3469 {
3470 /*
3471 * We can clear the inhibit force flag as even if we go back to the recompiler
3472 * without executing guest code in VT-x, the flag's condition to be cleared is
3473 * met and thus the cleared state is correct.
3474 */
3475 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3476 }
3477 }
3478
3479 /*
3480 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3481 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3482 * setting this would block host-NMIs and IRET will not clear the blocking.
3483 *
3484 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3485 */
3486 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3487 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3488 {
3489 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3490 }
3491
3492 return fIntrState;
3493}
3494
3495
3496/**
3497 * Exports the guest's interruptibility-state into the guest-state area in the
3498 * VMCS.
3499 *
3500 * @returns VBox status code.
3501 * @param pVCpu The cross context virtual CPU structure.
3502 * @param fIntrState The interruptibility-state to set.
3503 */
3504static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3505{
3506 NOREF(pVCpu);
3507 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3508 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3509 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3510}
3511
3512
3513/**
3514 * Exports the exception intercepts required for guest execution in the VMCS.
3515 *
3516 * @returns VBox status code.
3517 * @param pVCpu The cross context virtual CPU structure.
3518 *
3519 * @remarks No-long-jump zone!!!
3520 */
3521static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3522{
3523 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3524 {
3525 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3526 if (pVCpu->hm.s.fGIMTrapXcptUD)
3527 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3528#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3529 else
3530 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3531#endif
3532
3533 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3534 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3535
3536 /** @todo Optimize by checking cache before writing to VMCS. */
3537 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3538 AssertRCReturn(rc, rc);
3539
3540 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3541 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3542 }
3543 return VINF_SUCCESS;
3544}
3545
3546
3547/**
3548 * Exports the guest's RIP into the guest-state area in the VMCS.
3549 *
3550 * @returns VBox status code.
3551 * @param pVCpu The cross context virtual CPU structure.
3552 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3553 * out-of-sync. Make sure to update the required fields
3554 * before using them.
3555 *
3556 * @remarks No-long-jump zone!!!
3557 */
3558static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3559{
3560 int rc = VINF_SUCCESS;
3561 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3562 {
3563 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3564
3565 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3566 AssertRCReturn(rc, rc);
3567
3568 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3569 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3570 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3571 else
3572 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3573
3574 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3575 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3576 }
3577 return rc;
3578}
3579
3580
3581/**
3582 * Exports the guest's RSP into the guest-state area in the VMCS.
3583 *
3584 * @returns VBox status code.
3585 * @param pVCpu The cross context virtual CPU structure.
3586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3587 * out-of-sync. Make sure to update the required fields
3588 * before using them.
3589 *
3590 * @remarks No-long-jump zone!!!
3591 */
3592static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3593{
3594 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3595 {
3596 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3597
3598 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3599 AssertRCReturn(rc, rc);
3600
3601 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3602 }
3603 return VINF_SUCCESS;
3604}
3605
3606
3607/**
3608 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3609 *
3610 * @returns VBox status code.
3611 * @param pVCpu The cross context virtual CPU structure.
3612 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3613 * out-of-sync. Make sure to update the required fields
3614 * before using them.
3615 *
3616 * @remarks No-long-jump zone!!!
3617 */
3618static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3619{
3620 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3621 {
3622 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3623
3624 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3625 Let us assert it as such and use 32-bit VMWRITE. */
3626 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3627 X86EFLAGS fEFlags = pMixedCtx->eflags;
3628 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3629 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3630
3631 /*
3632 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3633 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3634 * can run the real-mode guest code under Virtual 8086 mode.
3635 */
3636 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3637 {
3638 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3639 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3640 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3641 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3642 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3643 }
3644
3645 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3646 AssertRCReturn(rc, rc);
3647
3648 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3649 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3650 }
3651 return VINF_SUCCESS;
3652}
3653
3654
3655/**
3656 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3657 *
3658 * The guest FPU state is always pre-loaded hence we don't need to bother about
3659 * sharing FPU related CR0 bits between the guest and host.
3660 *
3661 * @returns VBox status code.
3662 * @param pVCpu The cross context virtual CPU structure.
3663 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3664 * out-of-sync. Make sure to update the required fields
3665 * before using them.
3666 *
3667 * @remarks No-long-jump zone!!!
3668 */
3669static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3670{
3671 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3672 {
3673 PVM pVM = pVCpu->CTX_SUFF(pVM);
3674 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3675 Assert(!RT_HI_U32(pMixedCtx->cr0));
3676
3677 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3678 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3679
3680 /*
3681 * Setup VT-x's view of the guest CR0.
3682 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3683 */
3684 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3685 if (pVM->hm.s.fNestedPaging)
3686 {
3687 if (CPUMIsGuestPagingEnabled(pVCpu))
3688 {
3689 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3690 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3691 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3692 }
3693 else
3694 {
3695 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3696 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3697 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3698 }
3699
3700 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3701 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3702 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3703 }
3704 else
3705 {
3706 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3707 u32GuestCr0 |= X86_CR0_WP;
3708 }
3709
3710 /*
3711 * Guest FPU bits.
3712 *
3713 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3714 * using CR0.TS.
3715 *
3716 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3717 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3718 */
3719 u32GuestCr0 |= X86_CR0_NE;
3720
3721 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3722 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3723
3724 /*
3725 * Update exception intercepts.
3726 */
3727 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3728 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3729 {
3730 Assert(PDMVmmDevHeapIsEnabled(pVM));
3731 Assert(pVM->hm.s.vmx.pRealModeTSS);
3732 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3733 }
3734 else
3735 {
3736 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3737 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3738 if (fInterceptMF)
3739 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3740 }
3741
3742 /* Additional intercepts for debugging, define these yourself explicitly. */
3743#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3744 uXcptBitmap |= 0
3745 | RT_BIT(X86_XCPT_BP)
3746 | RT_BIT(X86_XCPT_DE)
3747 | RT_BIT(X86_XCPT_NM)
3748 | RT_BIT(X86_XCPT_TS)
3749 | RT_BIT(X86_XCPT_UD)
3750 | RT_BIT(X86_XCPT_NP)
3751 | RT_BIT(X86_XCPT_SS)
3752 | RT_BIT(X86_XCPT_GP)
3753 | RT_BIT(X86_XCPT_PF)
3754 | RT_BIT(X86_XCPT_MF)
3755 ;
3756#elif defined(HMVMX_ALWAYS_TRAP_PF)
3757 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3758#endif
3759 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3760 {
3761 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3762 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3763 }
3764 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3765
3766 /*
3767 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3768 */
3769 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3770 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3771 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3772 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3773 else
3774 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3775
3776 u32GuestCr0 |= fSetCr0;
3777 u32GuestCr0 &= fZapCr0;
3778 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3779
3780 /*
3781 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3782 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3783 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3784 */
3785 uint32_t u32Cr0Mask = X86_CR0_PE
3786 | X86_CR0_NE
3787 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3788 | X86_CR0_PG
3789 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3790 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3791 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3792
3793 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3794 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3795 * and @bugref{6944}. */
3796#if 0
3797 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3798 u32Cr0Mask &= ~X86_CR0_PE;
3799#endif
3800 /*
3801 * Finally, update VMCS fields with the CR0 values.
3802 */
3803 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3804 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3805 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3806 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3807 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3808 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3809 AssertRCReturn(rc, rc);
3810
3811 /* Update our caches. */
3812 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3813 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3814
3815 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3816
3817 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3818 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3819 }
3820
3821 return VINF_SUCCESS;
3822}
3823
3824
3825/**
3826 * Exports the guest control registers (CR3, CR4) into the guest-state area
3827 * in the VMCS.
3828 *
3829 * @returns VBox strict status code.
3830 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3831 * without unrestricted guest access and the VMMDev is not presently
3832 * mapped (e.g. EFI32).
3833 *
3834 * @param pVCpu The cross context virtual CPU structure.
3835 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3836 * out-of-sync. Make sure to update the required fields
3837 * before using them.
3838 *
3839 * @remarks No-long-jump zone!!!
3840 */
3841static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3842{
3843 int rc = VINF_SUCCESS;
3844 PVM pVM = pVCpu->CTX_SUFF(pVM);
3845
3846 /*
3847 * Guest CR2.
3848 * It's always loaded in the assembler code. Nothing to do here.
3849 */
3850
3851 /*
3852 * Guest CR3.
3853 */
3854 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3855 {
3856 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3857
3858 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3859 if (pVM->hm.s.fNestedPaging)
3860 {
3861 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3862
3863 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3864 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3865 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3866 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3867
3868 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3869 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3870 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3871
3872 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3873 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3874 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3875 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3876 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3877 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3878 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3879
3880 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3881 AssertRCReturn(rc, rc);
3882
3883 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3884 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3885 {
3886 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3887 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3888 {
3889 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3890 AssertRCReturn(rc, rc);
3891 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3892 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3893 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3894 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3895 AssertRCReturn(rc, rc);
3896 }
3897
3898 /*
3899 * The guest's view of its CR3 is unblemished with Nested Paging when the
3900 * guest is using paging or we have unrestricted guest execution to handle
3901 * the guest when it's not using paging.
3902 */
3903 GCPhysGuestCR3 = pMixedCtx->cr3;
3904 }
3905 else
3906 {
3907 /*
3908 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3909 * thinks it accesses physical memory directly, we use our identity-mapped
3910 * page table to map guest-linear to guest-physical addresses. EPT takes care
3911 * of translating it to host-physical addresses.
3912 */
3913 RTGCPHYS GCPhys;
3914 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3915
3916 /* We obtain it here every time as the guest could have relocated this PCI region. */
3917 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3918 if (RT_SUCCESS(rc))
3919 { /* likely */ }
3920 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3921 {
3922 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3923 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3924 }
3925 else
3926 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3927
3928 GCPhysGuestCR3 = GCPhys;
3929 }
3930
3931 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3932 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3933 AssertRCReturn(rc, rc);
3934 }
3935 else
3936 {
3937 /* Non-nested paging case, just use the hypervisor's CR3. */
3938 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3939
3940 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3941 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3942 AssertRCReturn(rc, rc);
3943 }
3944
3945 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3946 }
3947
3948 /*
3949 * Guest CR4.
3950 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3951 */
3952 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3953 {
3954 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3955 Assert(!RT_HI_U32(pMixedCtx->cr4));
3956
3957 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3958 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3959
3960 /*
3961 * Setup VT-x's view of the guest CR4.
3962 *
3963 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3964 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3965 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3966 *
3967 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3968 */
3969 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3970 {
3971 Assert(pVM->hm.s.vmx.pRealModeTSS);
3972 Assert(PDMVmmDevHeapIsEnabled(pVM));
3973 u32GuestCr4 &= ~X86_CR4_VME;
3974 }
3975
3976 if (pVM->hm.s.fNestedPaging)
3977 {
3978 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3979 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3980 {
3981 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3982 u32GuestCr4 |= X86_CR4_PSE;
3983 /* Our identity mapping is a 32-bit page directory. */
3984 u32GuestCr4 &= ~X86_CR4_PAE;
3985 }
3986 /* else use guest CR4.*/
3987 }
3988 else
3989 {
3990 /*
3991 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3992 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3993 */
3994 switch (pVCpu->hm.s.enmShadowMode)
3995 {
3996 case PGMMODE_REAL: /* Real-mode. */
3997 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3998 case PGMMODE_32_BIT: /* 32-bit paging. */
3999 {
4000 u32GuestCr4 &= ~X86_CR4_PAE;
4001 break;
4002 }
4003
4004 case PGMMODE_PAE: /* PAE paging. */
4005 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4006 {
4007 u32GuestCr4 |= X86_CR4_PAE;
4008 break;
4009 }
4010
4011 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4012 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4013#ifdef VBOX_ENABLE_64_BITS_GUESTS
4014 break;
4015#endif
4016 default:
4017 AssertFailed();
4018 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4019 }
4020 }
4021
4022 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4023 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4024 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4025 u32GuestCr4 |= fSetCr4;
4026 u32GuestCr4 &= fZapCr4;
4027
4028 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4029 that would cause a VM-exit. */
4030 uint32_t u32Cr4Mask = X86_CR4_VME
4031 | X86_CR4_PAE
4032 | X86_CR4_PGE
4033 | X86_CR4_PSE
4034 | X86_CR4_VMXE;
4035 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4036 u32Cr4Mask |= X86_CR4_OSXSAVE;
4037 if (pVM->cpum.ro.GuestFeatures.fPcid)
4038 u32Cr4Mask |= X86_CR4_PCIDE;
4039
4040 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4041 into the VMCS and update our cache. */
4042 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4043 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4044 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4045 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4046 AssertRCReturn(rc, rc);
4047 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4048
4049 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4050 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4051
4052 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4053
4054 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4055 fZapCr4));
4056 }
4057 return rc;
4058}
4059
4060
4061/**
4062 * Exports the guest debug registers into the guest-state area in the VMCS.
4063 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4064 *
4065 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4066 *
4067 * @returns VBox status code.
4068 * @param pVCpu The cross context virtual CPU structure.
4069 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4070 * out-of-sync. Make sure to update the required fields
4071 * before using them.
4072 *
4073 * @remarks No-long-jump zone!!!
4074 */
4075static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4076{
4077 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4078
4079#ifdef VBOX_STRICT
4080 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4081 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4082 {
4083 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4084 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4085 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4086 }
4087#endif
4088
4089 bool fSteppingDB = false;
4090 bool fInterceptMovDRx = false;
4091 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4092 if (pVCpu->hm.s.fSingleInstruction)
4093 {
4094 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4095 PVM pVM = pVCpu->CTX_SUFF(pVM);
4096 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4097 {
4098 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4099 Assert(fSteppingDB == false);
4100 }
4101 else
4102 {
4103 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4104 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4105 pVCpu->hm.s.fClearTrapFlag = true;
4106 fSteppingDB = true;
4107 }
4108 }
4109
4110 uint32_t u32GuestDr7;
4111 if ( fSteppingDB
4112 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4113 {
4114 /*
4115 * Use the combined guest and host DRx values found in the hypervisor register set
4116 * because the debugger has breakpoints active or someone is single stepping on the
4117 * host side without a monitor trap flag.
4118 *
4119 * Note! DBGF expects a clean DR6 state before executing guest code.
4120 */
4121#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4122 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4123 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4124 {
4125 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4126 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4127 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4128 }
4129 else
4130#endif
4131 if (!CPUMIsHyperDebugStateActive(pVCpu))
4132 {
4133 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4134 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4135 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4136 }
4137
4138 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4139 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4140 pVCpu->hm.s.fUsingHyperDR7 = true;
4141 fInterceptMovDRx = true;
4142 }
4143 else
4144 {
4145 /*
4146 * If the guest has enabled debug registers, we need to load them prior to
4147 * executing guest code so they'll trigger at the right time.
4148 */
4149 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4150 {
4151#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4152 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4153 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4154 {
4155 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4156 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4157 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4158 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4159 }
4160 else
4161#endif
4162 if (!CPUMIsGuestDebugStateActive(pVCpu))
4163 {
4164 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4165 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4166 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4167 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4168 }
4169 Assert(!fInterceptMovDRx);
4170 }
4171 /*
4172 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4173 * must intercept #DB in order to maintain a correct DR6 guest value, and
4174 * because we need to intercept it to prevent nested #DBs from hanging the
4175 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4176 */
4177#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4178 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4179 && !CPUMIsGuestDebugStateActive(pVCpu))
4180#else
4181 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4182#endif
4183 {
4184 fInterceptMovDRx = true;
4185 }
4186
4187 /* Update DR7 with the actual guest value. */
4188 u32GuestDr7 = pMixedCtx->dr[7];
4189 pVCpu->hm.s.fUsingHyperDR7 = false;
4190 }
4191
4192 if (fInterceptMovDRx)
4193 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4194 else
4195 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4196
4197 /*
4198 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4199 * monitor-trap flag and update our cache.
4200 */
4201 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4202 {
4203 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4204 AssertRCReturn(rc2, rc2);
4205 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4206 }
4207
4208 /*
4209 * Update guest DR7.
4210 */
4211 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4212 AssertRCReturn(rc, rc);
4213
4214 return VINF_SUCCESS;
4215}
4216
4217
4218#ifdef VBOX_STRICT
4219/**
4220 * Strict function to validate segment registers.
4221 *
4222 * @param pVCpu The cross context virtual CPU structure.
4223 * @param pCtx Pointer to the guest-CPU context.
4224 *
4225 * @remarks Will import guest CR0 on strict builds during validation of
4226 * segments.
4227 */
4228static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4229{
4230 /*
4231 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4232 *
4233 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4234 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4235 * and doesn't change the guest-context value.
4236 */
4237 PVM pVM = pVCpu->CTX_SUFF(pVM);
4238 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4239 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4240 && ( !CPUMIsGuestInRealModeEx(pCtx)
4241 && !CPUMIsGuestInV86ModeEx(pCtx)))
4242 {
4243 /* Protected mode checks */
4244 /* CS */
4245 Assert(pCtx->cs.Attr.n.u1Present);
4246 Assert(!(pCtx->cs.Attr.u & 0xf00));
4247 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4248 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4249 || !(pCtx->cs.Attr.n.u1Granularity));
4250 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4251 || (pCtx->cs.Attr.n.u1Granularity));
4252 /* CS cannot be loaded with NULL in protected mode. */
4253 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4254 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4255 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4256 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4257 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4258 else
4259 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4260 /* SS */
4261 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4262 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4263 if ( !(pCtx->cr0 & X86_CR0_PE)
4264 || pCtx->cs.Attr.n.u4Type == 3)
4265 {
4266 Assert(!pCtx->ss.Attr.n.u2Dpl);
4267 }
4268 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4269 {
4270 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4271 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4272 Assert(pCtx->ss.Attr.n.u1Present);
4273 Assert(!(pCtx->ss.Attr.u & 0xf00));
4274 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4275 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4276 || !(pCtx->ss.Attr.n.u1Granularity));
4277 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4278 || (pCtx->ss.Attr.n.u1Granularity));
4279 }
4280 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4281 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4282 {
4283 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4284 Assert(pCtx->ds.Attr.n.u1Present);
4285 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4286 Assert(!(pCtx->ds.Attr.u & 0xf00));
4287 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4288 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4289 || !(pCtx->ds.Attr.n.u1Granularity));
4290 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4291 || (pCtx->ds.Attr.n.u1Granularity));
4292 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4293 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4294 }
4295 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4296 {
4297 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4298 Assert(pCtx->es.Attr.n.u1Present);
4299 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4300 Assert(!(pCtx->es.Attr.u & 0xf00));
4301 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4302 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4303 || !(pCtx->es.Attr.n.u1Granularity));
4304 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4305 || (pCtx->es.Attr.n.u1Granularity));
4306 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4307 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4308 }
4309 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4310 {
4311 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4312 Assert(pCtx->fs.Attr.n.u1Present);
4313 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4314 Assert(!(pCtx->fs.Attr.u & 0xf00));
4315 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4316 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4317 || !(pCtx->fs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4319 || (pCtx->fs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4321 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4322 }
4323 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4324 {
4325 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4326 Assert(pCtx->gs.Attr.n.u1Present);
4327 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4328 Assert(!(pCtx->gs.Attr.u & 0xf00));
4329 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4330 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4331 || !(pCtx->gs.Attr.n.u1Granularity));
4332 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4333 || (pCtx->gs.Attr.n.u1Granularity));
4334 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4335 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4336 }
4337 /* 64-bit capable CPUs. */
4338# if HC_ARCH_BITS == 64
4339 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4340 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4341 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4342 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4343# endif
4344 }
4345 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4346 || ( CPUMIsGuestInRealModeEx(pCtx)
4347 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4348 {
4349 /* Real and v86 mode checks. */
4350 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4351 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4352 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4353 {
4354 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4355 }
4356 else
4357 {
4358 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4359 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4360 }
4361
4362 /* CS */
4363 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4364 Assert(pCtx->cs.u32Limit == 0xffff);
4365 Assert(u32CSAttr == 0xf3);
4366 /* SS */
4367 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4368 Assert(pCtx->ss.u32Limit == 0xffff);
4369 Assert(u32SSAttr == 0xf3);
4370 /* DS */
4371 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4372 Assert(pCtx->ds.u32Limit == 0xffff);
4373 Assert(u32DSAttr == 0xf3);
4374 /* ES */
4375 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4376 Assert(pCtx->es.u32Limit == 0xffff);
4377 Assert(u32ESAttr == 0xf3);
4378 /* FS */
4379 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4380 Assert(pCtx->fs.u32Limit == 0xffff);
4381 Assert(u32FSAttr == 0xf3);
4382 /* GS */
4383 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4384 Assert(pCtx->gs.u32Limit == 0xffff);
4385 Assert(u32GSAttr == 0xf3);
4386 /* 64-bit capable CPUs. */
4387# if HC_ARCH_BITS == 64
4388 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4389 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4390 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4391 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4392# endif
4393 }
4394}
4395#endif /* VBOX_STRICT */
4396
4397
4398/**
4399 * Exports a guest segment register into the guest-state area in the VMCS.
4400 *
4401 * @returns VBox status code.
4402 * @param pVCpu The cross context virtual CPU structure.
4403 * @param idxSel Index of the selector in the VMCS.
4404 * @param idxLimit Index of the segment limit in the VMCS.
4405 * @param idxBase Index of the segment base in the VMCS.
4406 * @param idxAccess Index of the access rights of the segment in the VMCS.
4407 * @param pSelReg Pointer to the segment selector.
4408 *
4409 * @remarks No-long-jump zone!!!
4410 */
4411static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4412 PCCPUMSELREG pSelReg)
4413{
4414 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4415 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4416 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4417 AssertRCReturn(rc, rc);
4418
4419 uint32_t u32Access = pSelReg->Attr.u;
4420 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4421 {
4422 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4423 u32Access = 0xf3;
4424 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4425 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4426 }
4427 else
4428 {
4429 /*
4430 * The way to differentiate between whether this is really a null selector or was just
4431 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4432 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4433 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4434 * NULL selectors loaded in protected-mode have their attribute as 0.
4435 */
4436 if (!u32Access)
4437 u32Access = X86DESCATTR_UNUSABLE;
4438 }
4439
4440 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4441 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4442 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4443
4444 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4445 AssertRCReturn(rc, rc);
4446 return rc;
4447}
4448
4449
4450/**
4451 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4452 * into the guest-state area in the VMCS.
4453 *
4454 * @returns VBox status code.
4455 * @param pVCpu The cross context virtual CPU structure.
4456 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4457 * out-of-sync. Make sure to update the required fields
4458 * before using them.
4459 *
4460 * @remarks Will import guest CR0 on strict builds during validation of
4461 * segments.
4462 * @remarks No-long-jump zone!!!
4463 */
4464static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4465{
4466 int rc = VERR_INTERNAL_ERROR_5;
4467 PVM pVM = pVCpu->CTX_SUFF(pVM);
4468
4469 /*
4470 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4471 */
4472 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4473 {
4474#ifdef VBOX_WITH_REM
4475 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4476 {
4477 Assert(pVM->hm.s.vmx.pRealModeTSS);
4478 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4479 if ( pVCpu->hm.s.vmx.fWasInRealMode
4480 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4481 {
4482 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4483 in real-mode (e.g. OpenBSD 4.0) */
4484 REMFlushTBs(pVM);
4485 Log4Func(("Switch to protected mode detected!\n"));
4486 pVCpu->hm.s.vmx.fWasInRealMode = false;
4487 }
4488 }
4489#endif
4490 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4491 {
4492 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4493 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4494 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4495 rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4496 AssertRCReturn(rc, rc);
4497 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4498 }
4499
4500 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4501 {
4502 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4503 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4504 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4505 rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4506 AssertRCReturn(rc, rc);
4507 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4508 }
4509
4510 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4511 {
4512 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4513 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4514 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4515 rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4516 AssertRCReturn(rc, rc);
4517 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4518 }
4519
4520 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4521 {
4522 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4523 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4524 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4525 rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4526 AssertRCReturn(rc, rc);
4527 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4528 }
4529
4530 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4531 {
4532 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4533 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4534 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4535 rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4536 AssertRCReturn(rc, rc);
4537 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4538 }
4539
4540 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4541 {
4542 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4543 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4544 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4545 rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4546 AssertRCReturn(rc, rc);
4547 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4548 }
4549
4550#ifdef VBOX_STRICT
4551 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4552#endif
4553
4554 /* Update the exit history entry with the correct CS.BASE + RIP. */
4555 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4556 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4557
4558 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4559 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4560 }
4561
4562 /*
4563 * Guest TR.
4564 */
4565 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4566 {
4567 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4568
4569 /*
4570 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4571 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4572 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4573 */
4574 uint16_t u16Sel = 0;
4575 uint32_t u32Limit = 0;
4576 uint64_t u64Base = 0;
4577 uint32_t u32AccessRights = 0;
4578
4579 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4580 {
4581 u16Sel = pMixedCtx->tr.Sel;
4582 u32Limit = pMixedCtx->tr.u32Limit;
4583 u64Base = pMixedCtx->tr.u64Base;
4584 u32AccessRights = pMixedCtx->tr.Attr.u;
4585 }
4586 else
4587 {
4588 Assert(pVM->hm.s.vmx.pRealModeTSS);
4589 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4590
4591 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4592 RTGCPHYS GCPhys;
4593 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4594 AssertRCReturn(rc, rc);
4595
4596 X86DESCATTR DescAttr;
4597 DescAttr.u = 0;
4598 DescAttr.n.u1Present = 1;
4599 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4600
4601 u16Sel = 0;
4602 u32Limit = HM_VTX_TSS_SIZE;
4603 u64Base = GCPhys; /* in real-mode phys = virt. */
4604 u32AccessRights = DescAttr.u;
4605 }
4606
4607 /* Validate. */
4608 Assert(!(u16Sel & RT_BIT(2)));
4609 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4610 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4611 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4612 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4613 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4614 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4615 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4616 Assert( (u32Limit & 0xfff) == 0xfff
4617 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4618 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4619 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4620
4621 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4622 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4623 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4624 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4625 AssertRCReturn(rc, rc);
4626
4627 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4628 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4629 }
4630
4631 /*
4632 * Guest GDTR.
4633 */
4634 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4635 {
4636 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4637
4638 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4639 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4640 AssertRCReturn(rc, rc);
4641
4642 /* Validate. */
4643 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4644
4645 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4646 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4647 }
4648
4649 /*
4650 * Guest LDTR.
4651 */
4652 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4653 {
4654 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4655
4656 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4657 uint32_t u32Access = 0;
4658 if (!pMixedCtx->ldtr.Attr.u)
4659 u32Access = X86DESCATTR_UNUSABLE;
4660 else
4661 u32Access = pMixedCtx->ldtr.Attr.u;
4662
4663 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4664 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4665 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4666 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4667 AssertRCReturn(rc, rc);
4668
4669 /* Validate. */
4670 if (!(u32Access & X86DESCATTR_UNUSABLE))
4671 {
4672 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4673 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4674 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4675 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4676 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4677 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4678 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4679 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4680 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4681 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4682 }
4683
4684 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4685 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4686 }
4687
4688 /*
4689 * Guest IDTR.
4690 */
4691 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4692 {
4693 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4694
4695 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4696 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4697 AssertRCReturn(rc, rc);
4698
4699 /* Validate. */
4700 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4701
4702 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4703 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4704 }
4705
4706 return VINF_SUCCESS;
4707}
4708
4709
4710/**
4711 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4712 * areas.
4713 *
4714 * These MSRs will automatically be loaded to the host CPU on every successful
4715 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4716 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4717 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4718 *
4719 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4720 *
4721 * @returns VBox status code.
4722 * @param pVCpu The cross context virtual CPU structure.
4723 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4724 * out-of-sync. Make sure to update the required fields
4725 * before using them.
4726 *
4727 * @remarks No-long-jump zone!!!
4728 */
4729static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4730{
4731 AssertPtr(pVCpu);
4732 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4733
4734 /*
4735 * MSRs that we use the auto-load/store MSR area in the VMCS.
4736 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4737 */
4738 PVM pVM = pVCpu->CTX_SUFF(pVM);
4739 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4740 {
4741 if (pVM->hm.s.fAllow64BitGuests)
4742 {
4743#if HC_ARCH_BITS == 32
4744 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4745
4746 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4747 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4748 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4749 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4750 AssertRCReturn(rc, rc);
4751# ifdef LOG_ENABLED
4752 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4753 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4754 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4755# endif
4756#endif
4757 }
4758 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4759 }
4760
4761 /*
4762 * Guest Sysenter MSRs.
4763 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4764 * VM-exits on WRMSRs for these MSRs.
4765 */
4766 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4767 {
4768 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4769
4770 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4771 {
4772 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4773 AssertRCReturn(rc, rc);
4774 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4775 }
4776
4777 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4778 {
4779 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4780 AssertRCReturn(rc, rc);
4781 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4782 }
4783
4784 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4785 {
4786 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4787 AssertRCReturn(rc, rc);
4788 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4789 }
4790 }
4791
4792 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4793 {
4794 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4795
4796 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4797 {
4798 /*
4799 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4800 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4801 */
4802 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4803 {
4804 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4805 AssertRCReturn(rc,rc);
4806 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4807 }
4808 else
4809 {
4810 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4811 NULL /* pfAddedAndUpdated */);
4812 AssertRCReturn(rc, rc);
4813
4814 /* We need to intercept reads too, see @bugref{7386#c16}. */
4815 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4816 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4817 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4818 pVCpu->hm.s.vmx.cMsrs));
4819 }
4820 }
4821 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4822 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4823 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4824 }
4825
4826 return VINF_SUCCESS;
4827}
4828
4829
4830#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4831/**
4832 * Check if guest state allows safe use of 32-bit switcher again.
4833 *
4834 * Segment bases and protected mode structures must be 32-bit addressable
4835 * because the 32-bit switcher will ignore high dword when writing these VMCS
4836 * fields. See @bugref{8432} for details.
4837 *
4838 * @returns true if safe, false if must continue to use the 64-bit switcher.
4839 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4840 * out-of-sync. Make sure to update the required fields
4841 * before using them.
4842 *
4843 * @remarks No-long-jump zone!!!
4844 */
4845static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4846{
4847 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4848 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4849 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4850 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4851 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4852 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4853 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4854 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4855 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4856 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4857
4858 /* All good, bases are 32-bit. */
4859 return true;
4860}
4861#endif
4862
4863
4864/**
4865 * Selects up the appropriate function to run guest code.
4866 *
4867 * @returns VBox status code.
4868 * @param pVCpu The cross context virtual CPU structure.
4869 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4870 * out-of-sync. Make sure to update the required fields
4871 * before using them.
4872 *
4873 * @remarks No-long-jump zone!!!
4874 */
4875static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4876{
4877 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4878 {
4879#ifndef VBOX_ENABLE_64_BITS_GUESTS
4880 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4881#endif
4882 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4883#if HC_ARCH_BITS == 32
4884 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4885 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4886 {
4887#ifdef VBOX_STRICT
4888 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4889 {
4890 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4891 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4892 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4893 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4894 | HM_CHANGED_VMX_ENTRY_CTLS
4895 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4896 }
4897#endif
4898 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4899
4900 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4901 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4902 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4903 Log4Func(("Selected 64-bit switcher\n"));
4904 }
4905#else
4906 /* 64-bit host. */
4907 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4908#endif
4909 }
4910 else
4911 {
4912 /* Guest is not in long mode, use the 32-bit handler. */
4913#if HC_ARCH_BITS == 32
4914 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4915 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4916 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4917 {
4918# ifdef VBOX_STRICT
4919 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4920 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4921 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4922 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4923 | HM_CHANGED_VMX_ENTRY_CTLS
4924 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4925# endif
4926 }
4927# ifdef VBOX_ENABLE_64_BITS_GUESTS
4928 /*
4929 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4930 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4931 * switcher flag because now we know the guest is in a sane state where it's safe
4932 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4933 * the much faster 32-bit switcher again.
4934 */
4935 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4936 {
4937 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4938 Log4Func(("Selected 32-bit switcher\n"));
4939 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4940 }
4941 else
4942 {
4943 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4944 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4945 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4946 {
4947 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4948 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4949 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4950 | HM_CHANGED_VMX_ENTRY_CTLS
4951 | HM_CHANGED_VMX_EXIT_CTLS
4952 | HM_CHANGED_HOST_CONTEXT);
4953 Log4Func(("Selected 32-bit switcher (safe)\n"));
4954 }
4955 }
4956# else
4957 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4958# endif
4959#else
4960 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4961#endif
4962 }
4963 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4964 return VINF_SUCCESS;
4965}
4966
4967
4968/**
4969 * Wrapper for running the guest code in VT-x.
4970 *
4971 * @returns VBox status code, no informational status codes.
4972 * @param pVCpu The cross context virtual CPU structure.
4973 * @param pCtx Pointer to the guest-CPU context.
4974 *
4975 * @remarks No-long-jump zone!!!
4976 */
4977DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4978{
4979 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4980 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4981
4982 /*
4983 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4984 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4985 * callee-saved and thus the need for this XMM wrapper.
4986 *
4987 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4988 */
4989 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4990 /** @todo Add stats for resume vs launch. */
4991 PVM pVM = pVCpu->CTX_SUFF(pVM);
4992#ifdef VBOX_WITH_KERNEL_USING_XMM
4993 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4994#else
4995 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4996#endif
4997 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4998 return rc;
4999}
5000
5001
5002/**
5003 * Reports world-switch error and dumps some useful debug info.
5004 *
5005 * @param pVCpu The cross context virtual CPU structure.
5006 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5007 * @param pCtx Pointer to the guest-CPU context.
5008 * @param pVmxTransient Pointer to the VMX transient structure (only
5009 * exitReason updated).
5010 */
5011static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5012{
5013 Assert(pVCpu);
5014 Assert(pCtx);
5015 Assert(pVmxTransient);
5016 HMVMX_ASSERT_PREEMPT_SAFE();
5017
5018 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5019 switch (rcVMRun)
5020 {
5021 case VERR_VMX_INVALID_VMXON_PTR:
5022 AssertFailed();
5023 break;
5024 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5025 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5026 {
5027 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5028 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5029 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5030 AssertRC(rc);
5031
5032 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5033 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5034 Cannot do it here as we may have been long preempted. */
5035
5036#ifdef VBOX_STRICT
5037 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5038 pVmxTransient->uExitReason));
5039 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5040 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5041 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5042 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5043 else
5044 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5045 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5046 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5047
5048 /* VMX control bits. */
5049 uint32_t u32Val;
5050 uint64_t u64Val;
5051 RTHCUINTREG uHCReg;
5052 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5053 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5054 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5055 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5056 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5057 {
5058 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5059 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5060 }
5061 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5062 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5063 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5064 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5065 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5066 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5067 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5068 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5069 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5070 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5071 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5072 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5073 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5074 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5075 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5076 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5077 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5078 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5079 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5080 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5081 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5082 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5083 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5084 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5085 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5086 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5088 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5089 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5090 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5091 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5092 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5093 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5094 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5095 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5096 {
5097 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5098 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5099 }
5100
5101 /* Guest bits. */
5102 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5103 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5104 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5105 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5106 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5107 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5108 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5109 {
5110 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5111 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5112 }
5113
5114 /* Host bits. */
5115 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5116 Log4(("Host CR0 %#RHr\n", uHCReg));
5117 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5118 Log4(("Host CR3 %#RHr\n", uHCReg));
5119 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5120 Log4(("Host CR4 %#RHr\n", uHCReg));
5121
5122 RTGDTR HostGdtr;
5123 PCX86DESCHC pDesc;
5124 ASMGetGDTR(&HostGdtr);
5125 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5126 Log4(("Host CS %#08x\n", u32Val));
5127 if (u32Val < HostGdtr.cbGdt)
5128 {
5129 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5130 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5131 }
5132
5133 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5134 Log4(("Host DS %#08x\n", u32Val));
5135 if (u32Val < HostGdtr.cbGdt)
5136 {
5137 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5138 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5139 }
5140
5141 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5142 Log4(("Host ES %#08x\n", u32Val));
5143 if (u32Val < HostGdtr.cbGdt)
5144 {
5145 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5146 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5147 }
5148
5149 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5150 Log4(("Host FS %#08x\n", u32Val));
5151 if (u32Val < HostGdtr.cbGdt)
5152 {
5153 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5154 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5155 }
5156
5157 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5158 Log4(("Host GS %#08x\n", u32Val));
5159 if (u32Val < HostGdtr.cbGdt)
5160 {
5161 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5162 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5163 }
5164
5165 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5166 Log4(("Host SS %#08x\n", u32Val));
5167 if (u32Val < HostGdtr.cbGdt)
5168 {
5169 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5170 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5171 }
5172
5173 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5174 Log4(("Host TR %#08x\n", u32Val));
5175 if (u32Val < HostGdtr.cbGdt)
5176 {
5177 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5178 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5179 }
5180
5181 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5182 Log4(("Host TR Base %#RHv\n", uHCReg));
5183 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5184 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5185 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5186 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5187 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5188 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5189 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5190 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5191 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5192 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5193 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5194 Log4(("Host RSP %#RHv\n", uHCReg));
5195 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5196 Log4(("Host RIP %#RHv\n", uHCReg));
5197# if HC_ARCH_BITS == 64
5198 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5199 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5200 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5201 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5202 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5203 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5204# endif
5205#endif /* VBOX_STRICT */
5206 break;
5207 }
5208
5209 default:
5210 /* Impossible */
5211 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5212 break;
5213 }
5214 NOREF(pCtx);
5215}
5216
5217
5218#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5219#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5220# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5221#endif
5222#ifdef VBOX_STRICT
5223static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5224{
5225 switch (idxField)
5226 {
5227 case VMX_VMCS_GUEST_RIP:
5228 case VMX_VMCS_GUEST_RSP:
5229 case VMX_VMCS_GUEST_SYSENTER_EIP:
5230 case VMX_VMCS_GUEST_SYSENTER_ESP:
5231 case VMX_VMCS_GUEST_GDTR_BASE:
5232 case VMX_VMCS_GUEST_IDTR_BASE:
5233 case VMX_VMCS_GUEST_CS_BASE:
5234 case VMX_VMCS_GUEST_DS_BASE:
5235 case VMX_VMCS_GUEST_ES_BASE:
5236 case VMX_VMCS_GUEST_FS_BASE:
5237 case VMX_VMCS_GUEST_GS_BASE:
5238 case VMX_VMCS_GUEST_SS_BASE:
5239 case VMX_VMCS_GUEST_LDTR_BASE:
5240 case VMX_VMCS_GUEST_TR_BASE:
5241 case VMX_VMCS_GUEST_CR3:
5242 return true;
5243 }
5244 return false;
5245}
5246
5247static bool hmR0VmxIsValidReadField(uint32_t idxField)
5248{
5249 switch (idxField)
5250 {
5251 /* Read-only fields. */
5252 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5253 return true;
5254 }
5255 /* Remaining readable fields should also be writable. */
5256 return hmR0VmxIsValidWriteField(idxField);
5257}
5258#endif /* VBOX_STRICT */
5259
5260
5261/**
5262 * Executes the specified handler in 64-bit mode.
5263 *
5264 * @returns VBox status code (no informational status codes).
5265 * @param pVCpu The cross context virtual CPU structure.
5266 * @param enmOp The operation to perform.
5267 * @param cParams Number of parameters.
5268 * @param paParam Array of 32-bit parameters.
5269 */
5270VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5271{
5272 PVM pVM = pVCpu->CTX_SUFF(pVM);
5273 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5274 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5275 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5276 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5277
5278#ifdef VBOX_STRICT
5279 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5280 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5281
5282 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5283 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5284#endif
5285
5286 /* Disable interrupts. */
5287 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5288
5289#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5290 RTCPUID idHostCpu = RTMpCpuId();
5291 CPUMR0SetLApic(pVCpu, idHostCpu);
5292#endif
5293
5294 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5295 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5296
5297 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5298 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5299 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5300
5301 /* Leave VMX Root Mode. */
5302 VMXDisable();
5303
5304 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5305
5306 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5307 CPUMSetHyperEIP(pVCpu, enmOp);
5308 for (int i = (int)cParams - 1; i >= 0; i--)
5309 CPUMPushHyper(pVCpu, paParam[i]);
5310
5311 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5312
5313 /* Call the switcher. */
5314 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5315 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5316
5317 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5318 /* Make sure the VMX instructions don't cause #UD faults. */
5319 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5320
5321 /* Re-enter VMX Root Mode */
5322 int rc2 = VMXEnable(HCPhysCpuPage);
5323 if (RT_FAILURE(rc2))
5324 {
5325 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5326 ASMSetFlags(fOldEFlags);
5327 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5328 return rc2;
5329 }
5330
5331 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5332 AssertRC(rc2);
5333 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5334 Assert(!(ASMGetFlags() & X86_EFL_IF));
5335 ASMSetFlags(fOldEFlags);
5336 return rc;
5337}
5338
5339
5340/**
5341 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5342 * supporting 64-bit guests.
5343 *
5344 * @returns VBox status code.
5345 * @param fResume Whether to VMLAUNCH or VMRESUME.
5346 * @param pCtx Pointer to the guest-CPU context.
5347 * @param pCache Pointer to the VMCS cache.
5348 * @param pVM The cross context VM structure.
5349 * @param pVCpu The cross context virtual CPU structure.
5350 */
5351DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5352{
5353 NOREF(fResume);
5354
5355 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5356 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5357
5358#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5359 pCache->uPos = 1;
5360 pCache->interPD = PGMGetInterPaeCR3(pVM);
5361 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5362#endif
5363
5364#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5365 pCache->TestIn.HCPhysCpuPage = 0;
5366 pCache->TestIn.HCPhysVmcs = 0;
5367 pCache->TestIn.pCache = 0;
5368 pCache->TestOut.HCPhysVmcs = 0;
5369 pCache->TestOut.pCache = 0;
5370 pCache->TestOut.pCtx = 0;
5371 pCache->TestOut.eflags = 0;
5372#else
5373 NOREF(pCache);
5374#endif
5375
5376 uint32_t aParam[10];
5377 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5378 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5379 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5380 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5381 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5382 aParam[5] = 0;
5383 aParam[6] = VM_RC_ADDR(pVM, pVM);
5384 aParam[7] = 0;
5385 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5386 aParam[9] = 0;
5387
5388#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5389 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5390 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5391#endif
5392 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5393
5394#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5395 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5396 Assert(pCtx->dr[4] == 10);
5397 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5398#endif
5399
5400#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5401 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5402 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5403 pVCpu->hm.s.vmx.HCPhysVmcs));
5404 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5405 pCache->TestOut.HCPhysVmcs));
5406 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5407 pCache->TestOut.pCache));
5408 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5409 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5410 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5411 pCache->TestOut.pCtx));
5412 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5413#endif
5414 NOREF(pCtx);
5415 return rc;
5416}
5417
5418
5419/**
5420 * Initialize the VMCS-Read cache.
5421 *
5422 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5423 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5424 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5425 * (those that have a 32-bit FULL & HIGH part).
5426 *
5427 * @returns VBox status code.
5428 * @param pVCpu The cross context virtual CPU structure.
5429 */
5430static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5431{
5432#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5433 do { \
5434 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5435 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5436 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5437 ++cReadFields; \
5438 } while (0)
5439
5440 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5441 uint32_t cReadFields = 0;
5442
5443 /*
5444 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5445 * and serve to indicate exceptions to the rules.
5446 */
5447
5448 /* Guest-natural selector base fields. */
5449#if 0
5450 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5453#endif
5454 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5466#if 0
5467 /* Unused natural width guest-state fields. */
5468 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5470#endif
5471 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5473
5474 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5475 these 64-bit fields (using "FULL" and "HIGH" fields). */
5476#if 0
5477 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5486#endif
5487
5488 /* Natural width guest-state fields. */
5489 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5490#if 0
5491 /* Currently unused field. */
5492 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5493#endif
5494
5495 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5496 {
5497 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5498 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5499 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5500 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5501 }
5502 else
5503 {
5504 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5505 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5506 }
5507
5508#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5509 return VINF_SUCCESS;
5510}
5511
5512
5513/**
5514 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5515 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5516 * darwin, running 64-bit guests).
5517 *
5518 * @returns VBox status code.
5519 * @param pVCpu The cross context virtual CPU structure.
5520 * @param idxField The VMCS field encoding.
5521 * @param u64Val 16, 32 or 64-bit value.
5522 */
5523VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5524{
5525 int rc;
5526 switch (idxField)
5527 {
5528 /*
5529 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5530 */
5531 /* 64-bit Control fields. */
5532 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5533 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5534 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5535 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5536 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5537 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5538 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5539 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5540 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5541 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5542 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5543 case VMX_VMCS64_CTRL_EPTP_FULL:
5544 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5545 /* 64-bit Guest-state fields. */
5546 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5547 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5548 case VMX_VMCS64_GUEST_PAT_FULL:
5549 case VMX_VMCS64_GUEST_EFER_FULL:
5550 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5551 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5552 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5553 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5554 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5555 /* 64-bit Host-state fields. */
5556 case VMX_VMCS64_HOST_PAT_FULL:
5557 case VMX_VMCS64_HOST_EFER_FULL:
5558 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5559 {
5560 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5561 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5562 break;
5563 }
5564
5565 /*
5566 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5567 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5568 */
5569 /* Natural-width Guest-state fields. */
5570 case VMX_VMCS_GUEST_CR3:
5571 case VMX_VMCS_GUEST_ES_BASE:
5572 case VMX_VMCS_GUEST_CS_BASE:
5573 case VMX_VMCS_GUEST_SS_BASE:
5574 case VMX_VMCS_GUEST_DS_BASE:
5575 case VMX_VMCS_GUEST_FS_BASE:
5576 case VMX_VMCS_GUEST_GS_BASE:
5577 case VMX_VMCS_GUEST_LDTR_BASE:
5578 case VMX_VMCS_GUEST_TR_BASE:
5579 case VMX_VMCS_GUEST_GDTR_BASE:
5580 case VMX_VMCS_GUEST_IDTR_BASE:
5581 case VMX_VMCS_GUEST_RSP:
5582 case VMX_VMCS_GUEST_RIP:
5583 case VMX_VMCS_GUEST_SYSENTER_ESP:
5584 case VMX_VMCS_GUEST_SYSENTER_EIP:
5585 {
5586 if (!(RT_HI_U32(u64Val)))
5587 {
5588 /* If this field is 64-bit, VT-x will zero out the top bits. */
5589 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5590 }
5591 else
5592 {
5593 /* Assert that only the 32->64 switcher case should ever come here. */
5594 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5595 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5596 }
5597 break;
5598 }
5599
5600 default:
5601 {
5602 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5603 rc = VERR_INVALID_PARAMETER;
5604 break;
5605 }
5606 }
5607 AssertRCReturn(rc, rc);
5608 return rc;
5609}
5610
5611
5612/**
5613 * Queue up a VMWRITE by using the VMCS write cache.
5614 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5615 *
5616 * @param pVCpu The cross context virtual CPU structure.
5617 * @param idxField The VMCS field encoding.
5618 * @param u64Val 16, 32 or 64-bit value.
5619 */
5620VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5621{
5622 AssertPtr(pVCpu);
5623 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5624
5625 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5626 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5627
5628 /* Make sure there are no duplicates. */
5629 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5630 {
5631 if (pCache->Write.aField[i] == idxField)
5632 {
5633 pCache->Write.aFieldVal[i] = u64Val;
5634 return VINF_SUCCESS;
5635 }
5636 }
5637
5638 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5639 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5640 pCache->Write.cValidEntries++;
5641 return VINF_SUCCESS;
5642}
5643#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5644
5645
5646/**
5647 * Sets up the usage of TSC-offsetting and updates the VMCS.
5648 *
5649 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5650 * VMX preemption timer.
5651 *
5652 * @returns VBox status code.
5653 * @param pVCpu The cross context virtual CPU structure.
5654 *
5655 * @remarks No-long-jump zone!!!
5656 */
5657static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5658{
5659 bool fOffsettedTsc;
5660 bool fParavirtTsc;
5661 PVM pVM = pVCpu->CTX_SUFF(pVM);
5662 uint64_t uTscOffset;
5663 if (pVM->hm.s.vmx.fUsePreemptTimer)
5664 {
5665 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5666
5667 /* Make sure the returned values have sane upper and lower boundaries. */
5668 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5669 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5670 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5671 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5672
5673 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5674 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5675 AssertRC(rc);
5676 }
5677 else
5678 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5679
5680 /** @todo later optimize this to be done elsewhere and not before every
5681 * VM-entry. */
5682 if (fParavirtTsc)
5683 {
5684 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5685 information before every VM-entry, hence disable it for performance sake. */
5686#if 0
5687 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5688 AssertRC(rc);
5689#endif
5690 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5691 }
5692
5693 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5694 if ( fOffsettedTsc
5695 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5696 {
5697 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5698 {
5699 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5700 AssertRC(rc);
5701 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5702 }
5703
5704 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5705 {
5706 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5707 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5708 AssertRC(rc);
5709 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5710 }
5711 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5712 }
5713 else
5714 {
5715 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5716 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5717 {
5718 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5719 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5720 AssertRC(rc);
5721 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5722 }
5723 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5724 }
5725}
5726
5727
5728/**
5729 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5730 * VM-exit interruption info type.
5731 *
5732 * @returns The IEM exception flags.
5733 * @param uVector The event vector.
5734 * @param uVmxVectorType The VMX event type.
5735 *
5736 * @remarks This function currently only constructs flags required for
5737 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5738 * and CR2 aspects of an exception are not included).
5739 */
5740static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5741{
5742 uint32_t fIemXcptFlags;
5743 switch (uVmxVectorType)
5744 {
5745 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5746 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5747 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5748 break;
5749
5750 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5751 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5752 break;
5753
5754 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5755 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5756 break;
5757
5758 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5759 {
5760 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5761 if (uVector == X86_XCPT_BP)
5762 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5763 else if (uVector == X86_XCPT_OF)
5764 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5765 else
5766 {
5767 fIemXcptFlags = 0;
5768 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5769 }
5770 break;
5771 }
5772
5773 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5774 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5775 break;
5776
5777 default:
5778 fIemXcptFlags = 0;
5779 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5780 break;
5781 }
5782 return fIemXcptFlags;
5783}
5784
5785
5786/**
5787 * Sets an event as a pending event to be injected into the guest.
5788 *
5789 * @param pVCpu The cross context virtual CPU structure.
5790 * @param u32IntInfo The VM-entry interruption-information field.
5791 * @param cbInstr The VM-entry instruction length in bytes (for software
5792 * interrupts, exceptions and privileged software
5793 * exceptions).
5794 * @param u32ErrCode The VM-entry exception error code.
5795 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5796 * page-fault.
5797 *
5798 * @remarks Statistics counter assumes this is a guest event being injected or
5799 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5800 * always incremented.
5801 */
5802DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5803 RTGCUINTPTR GCPtrFaultAddress)
5804{
5805 Assert(!pVCpu->hm.s.Event.fPending);
5806 pVCpu->hm.s.Event.fPending = true;
5807 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5808 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5809 pVCpu->hm.s.Event.cbInstr = cbInstr;
5810 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5811}
5812
5813
5814/**
5815 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5816 *
5817 * @param pVCpu The cross context virtual CPU structure.
5818 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5819 * out-of-sync. Make sure to update the required fields
5820 * before using them.
5821 */
5822DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5823{
5824 NOREF(pMixedCtx);
5825 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5826 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5827 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5828 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5829}
5830
5831
5832/**
5833 * Handle a condition that occurred while delivering an event through the guest
5834 * IDT.
5835 *
5836 * @returns Strict VBox status code (i.e. informational status codes too).
5837 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5838 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5839 * to continue execution of the guest which will delivery the \#DF.
5840 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5841 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5842 *
5843 * @param pVCpu The cross context virtual CPU structure.
5844 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5845 * out-of-sync. Make sure to update the required fields
5846 * before using them.
5847 * @param pVmxTransient Pointer to the VMX transient structure.
5848 *
5849 * @remarks No-long-jump zone!!!
5850 */
5851static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5852{
5853 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5854
5855 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5856 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5857 AssertRCReturn(rc2, rc2);
5858
5859 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5860 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5861 {
5862 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5863 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5864
5865 /*
5866 * If the event was a software interrupt (generated with INT n) or a software exception
5867 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5868 * can handle the VM-exit and continue guest execution which will re-execute the
5869 * instruction rather than re-injecting the exception, as that can cause premature
5870 * trips to ring-3 before injection and involve TRPM which currently has no way of
5871 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5872 * the problem).
5873 */
5874 IEMXCPTRAISE enmRaise;
5875 IEMXCPTRAISEINFO fRaiseInfo;
5876 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5877 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5878 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5879 {
5880 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5881 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5882 }
5883 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5884 {
5885 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5886 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5887 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5888 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5889 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5890 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5891 uExitVectorType), VERR_VMX_IPE_5);
5892
5893 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5894
5895 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5896 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5897 {
5898 pVmxTransient->fVectoringPF = true;
5899 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5900 }
5901 }
5902 else
5903 {
5904 /*
5905 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5906 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5907 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5908 */
5909 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5910 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5911 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5912 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5913 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5914 }
5915
5916 /*
5917 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5918 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5919 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5920 * subsequent VM-entry would fail.
5921 *
5922 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5923 */
5924 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5925 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5926 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5927 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5928 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5929 {
5930 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5931 }
5932
5933 switch (enmRaise)
5934 {
5935 case IEMXCPTRAISE_CURRENT_XCPT:
5936 {
5937 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5938 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5939 Assert(rcStrict == VINF_SUCCESS);
5940 break;
5941 }
5942
5943 case IEMXCPTRAISE_PREV_EVENT:
5944 {
5945 uint32_t u32ErrCode;
5946 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5947 {
5948 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5949 AssertRCReturn(rc2, rc2);
5950 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5951 }
5952 else
5953 u32ErrCode = 0;
5954
5955 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5956 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5957 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5958 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5959
5960 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5961 pVCpu->hm.s.Event.u32ErrCode));
5962 Assert(rcStrict == VINF_SUCCESS);
5963 break;
5964 }
5965
5966 case IEMXCPTRAISE_REEXEC_INSTR:
5967 Assert(rcStrict == VINF_SUCCESS);
5968 break;
5969
5970 case IEMXCPTRAISE_DOUBLE_FAULT:
5971 {
5972 /*
5973 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5974 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5975 */
5976 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5977 {
5978 pVmxTransient->fVectoringDoublePF = true;
5979 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5980 pMixedCtx->cr2));
5981 rcStrict = VINF_SUCCESS;
5982 }
5983 else
5984 {
5985 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5986 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5987 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5988 uIdtVector, uExitVector));
5989 rcStrict = VINF_HM_DOUBLE_FAULT;
5990 }
5991 break;
5992 }
5993
5994 case IEMXCPTRAISE_TRIPLE_FAULT:
5995 {
5996 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5997 rcStrict = VINF_EM_RESET;
5998 break;
5999 }
6000
6001 case IEMXCPTRAISE_CPU_HANG:
6002 {
6003 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6004 rcStrict = VERR_EM_GUEST_CPU_HANG;
6005 break;
6006 }
6007
6008 default:
6009 {
6010 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6011 rcStrict = VERR_VMX_IPE_2;
6012 break;
6013 }
6014 }
6015 }
6016 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6017 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6018 && uExitVector != X86_XCPT_DF
6019 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6020 {
6021 /*
6022 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6023 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6024 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6025 */
6026 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6027 {
6028 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6029 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6030 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6031 }
6032 }
6033
6034 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6035 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6036 return rcStrict;
6037}
6038
6039
6040/**
6041 * Imports a guest segment register from the current VMCS into
6042 * the guest-CPU context.
6043 *
6044 * @returns VBox status code.
6045 * @param pVCpu The cross context virtual CPU structure.
6046 * @param idxSel Index of the selector in the VMCS.
6047 * @param idxLimit Index of the segment limit in the VMCS.
6048 * @param idxBase Index of the segment base in the VMCS.
6049 * @param idxAccess Index of the access rights of the segment in the VMCS.
6050 * @param pSelReg Pointer to the segment selector.
6051 *
6052 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6053 * do not log!
6054 *
6055 * @remarks Never call this function directly!!! Use the
6056 * HMVMX_IMPORT_SREG() macro as that takes care
6057 * of whether to read from the VMCS cache or not.
6058 */
6059static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6060 PCPUMSELREG pSelReg)
6061{
6062 NOREF(pVCpu);
6063
6064 uint32_t u32Sel;
6065 uint32_t u32Limit;
6066 uint32_t u32Attr;
6067 uint64_t u64Base;
6068 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6069 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6070 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6071 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6072 AssertRCReturn(rc, rc);
6073
6074 pSelReg->Sel = (uint16_t)u32Sel;
6075 pSelReg->ValidSel = (uint16_t)u32Sel;
6076 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6077 pSelReg->u32Limit = u32Limit;
6078 pSelReg->u64Base = u64Base;
6079 pSelReg->Attr.u = u32Attr;
6080
6081 /*
6082 * If VT-x marks the segment as unusable, most other bits remain undefined:
6083 * - For CS the L, D and G bits have meaning.
6084 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6085 * - For the remaining data segments no bits are defined.
6086 *
6087 * The present bit and the unusable bit has been observed to be set at the
6088 * same time (the selector was supposed to be invalid as we started executing
6089 * a V8086 interrupt in ring-0).
6090 *
6091 * What should be important for the rest of the VBox code, is that the P bit is
6092 * cleared. Some of the other VBox code recognizes the unusable bit, but
6093 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6094 * safe side here, we'll strip off P and other bits we don't care about. If
6095 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6096 *
6097 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6098 */
6099 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6100 {
6101 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6102
6103 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6104 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6105 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6106#ifdef VBOX_STRICT
6107 VMMRZCallRing3Disable(pVCpu);
6108 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6109# ifdef DEBUG_bird
6110 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6111 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6112 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6113# endif
6114 VMMRZCallRing3Enable(pVCpu);
6115#endif
6116 }
6117 return VINF_SUCCESS;
6118}
6119
6120
6121/**
6122 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6123 *
6124 * @returns VBox status code.
6125 * @param pVCpu The cross context virtual CPU structure.
6126 *
6127 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6128 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6129 * instead!!!
6130 */
6131DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6132{
6133 uint64_t u64Val;
6134 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6135 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6136 {
6137 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6138 if (RT_SUCCESS(rc))
6139 {
6140 pCtx->rip = u64Val;
6141 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6142 }
6143 return rc;
6144 }
6145 return VINF_SUCCESS;
6146}
6147
6148
6149/**
6150 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6151 *
6152 * @returns VBox status code.
6153 * @param pVCpu The cross context virtual CPU structure.
6154 *
6155 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6156 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6157 * instead!!!
6158 */
6159DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6160{
6161 uint32_t u32Val;
6162 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6163 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6164 {
6165 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6166 if (RT_SUCCESS(rc))
6167 {
6168 pCtx->eflags.u32 = u32Val;
6169
6170 /* Restore eflags for real-on-v86-mode hack. */
6171 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6172 {
6173 pCtx->eflags.Bits.u1VM = 0;
6174 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6175 }
6176 }
6177 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6178 return rc;
6179 }
6180 return VINF_SUCCESS;
6181}
6182
6183
6184/**
6185 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6186 * context.
6187 *
6188 * @returns VBox status code.
6189 * @param pVCpu The cross context virtual CPU structure.
6190 *
6191 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6192 * do not log!
6193 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6194 * instead!!!
6195 */
6196DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6197{
6198 uint32_t u32Val;
6199 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6200 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6201 if (RT_SUCCESS(rc))
6202 {
6203 /*
6204 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6205 * might need them in hmR0VmxEvaluatePendingEvent().
6206 */
6207 if (!u32Val)
6208 {
6209 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6210 {
6211 rc = hmR0VmxImportGuestRip(pVCpu);
6212 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6213 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6214 }
6215
6216 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6217 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6218 }
6219 else
6220 {
6221 rc = hmR0VmxImportGuestRip(pVCpu);
6222 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6223
6224 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6225 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6226 {
6227 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6228 }
6229 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6230 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6231
6232 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6233 {
6234 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6235 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6236 }
6237 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6238 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6239 }
6240 }
6241 return rc;
6242}
6243
6244
6245/**
6246 * Worker for VMXR0ImportStateOnDemand.
6247 *
6248 * @returns VBox status code.
6249 * @param pVCpu The cross context virtual CPU structure.
6250 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6251 */
6252static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6253{
6254#define VMXLOCAL_BREAK_RC(a_rc) \
6255 if (RT_FAILURE(a_rc)) \
6256 break
6257
6258 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6259
6260 int rc = VINF_SUCCESS;
6261 PVM pVM = pVCpu->CTX_SUFF(pVM);
6262 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6263 uint64_t u64Val;
6264 uint32_t u32Val;
6265
6266 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6267
6268 /*
6269 * We disable interrupts to make the updating of the state and in particular
6270 * the fExtrn modification atomic wrt to preemption hooks.
6271 */
6272 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6273
6274 fWhat &= pCtx->fExtrn;
6275 if (fWhat)
6276 {
6277 do
6278 {
6279 if (fWhat & CPUMCTX_EXTRN_RIP)
6280 {
6281 rc = hmR0VmxImportGuestRip(pVCpu);
6282 VMXLOCAL_BREAK_RC(rc);
6283 }
6284
6285 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6286 {
6287 rc = hmR0VmxImportGuestRFlags(pVCpu);
6288 VMXLOCAL_BREAK_RC(rc);
6289 }
6290
6291 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6292 {
6293 rc = hmR0VmxImportGuestIntrState(pVCpu);
6294 VMXLOCAL_BREAK_RC(rc);
6295 }
6296
6297 if (fWhat & CPUMCTX_EXTRN_RSP)
6298 {
6299 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6300 VMXLOCAL_BREAK_RC(rc);
6301 pCtx->rsp = u64Val;
6302 }
6303
6304 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6305 {
6306 if (fWhat & CPUMCTX_EXTRN_CS)
6307 {
6308 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6309 VMXLOCAL_BREAK_RC(rc);
6310 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6311 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6312 }
6313 if (fWhat & CPUMCTX_EXTRN_SS)
6314 {
6315 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6316 VMXLOCAL_BREAK_RC(rc);
6317 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6318 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6319 }
6320 if (fWhat & CPUMCTX_EXTRN_DS)
6321 {
6322 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6323 VMXLOCAL_BREAK_RC(rc);
6324 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6325 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6326 }
6327 if (fWhat & CPUMCTX_EXTRN_ES)
6328 {
6329 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6330 VMXLOCAL_BREAK_RC(rc);
6331 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6332 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6333 }
6334 if (fWhat & CPUMCTX_EXTRN_FS)
6335 {
6336 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6337 VMXLOCAL_BREAK_RC(rc);
6338 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6339 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6340 }
6341 if (fWhat & CPUMCTX_EXTRN_GS)
6342 {
6343 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6344 VMXLOCAL_BREAK_RC(rc);
6345 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6346 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6347 }
6348 }
6349
6350 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6351 {
6352 if (fWhat & CPUMCTX_EXTRN_LDTR)
6353 {
6354 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6355 VMXLOCAL_BREAK_RC(rc);
6356 }
6357
6358 if (fWhat & CPUMCTX_EXTRN_GDTR)
6359 {
6360 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6361 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6362 VMXLOCAL_BREAK_RC(rc);
6363 pCtx->gdtr.pGdt = u64Val;
6364 pCtx->gdtr.cbGdt = u32Val;
6365 }
6366
6367 /* Guest IDTR. */
6368 if (fWhat & CPUMCTX_EXTRN_IDTR)
6369 {
6370 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6371 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6372 VMXLOCAL_BREAK_RC(rc);
6373 pCtx->idtr.pIdt = u64Val;
6374 pCtx->idtr.cbIdt = u32Val;
6375 }
6376
6377 /* Guest TR. */
6378 if (fWhat & CPUMCTX_EXTRN_TR)
6379 {
6380 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6381 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6382 {
6383 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6384 VMXLOCAL_BREAK_RC(rc);
6385 }
6386 }
6387 }
6388
6389 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6390 {
6391 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6392 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6393 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6394 pCtx->SysEnter.cs = u32Val;
6395 VMXLOCAL_BREAK_RC(rc);
6396 }
6397
6398#if HC_ARCH_BITS == 64
6399 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6400 {
6401 if ( pVM->hm.s.fAllow64BitGuests
6402 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6403 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6404 }
6405
6406 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6407 {
6408 if ( pVM->hm.s.fAllow64BitGuests
6409 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6410 {
6411 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6412 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6413 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6414 }
6415 }
6416#endif
6417
6418 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6419#if HC_ARCH_BITS == 32
6420 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6421#endif
6422 )
6423 {
6424 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6425 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6426 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6427 {
6428 switch (pMsr->u32Msr)
6429 {
6430#if HC_ARCH_BITS == 32
6431 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6432 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6433 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6434 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6435#endif
6436 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6437 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6438 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6439 default:
6440 {
6441 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6442 ASMSetFlags(fEFlags);
6443 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6444 cMsrs));
6445 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6446 }
6447 }
6448 }
6449 }
6450
6451 if (fWhat & CPUMCTX_EXTRN_DR7)
6452 {
6453 if (!pVCpu->hm.s.fUsingHyperDR7)
6454 {
6455 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6456 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6457 VMXLOCAL_BREAK_RC(rc);
6458 pCtx->dr[7] = u32Val;
6459 }
6460 }
6461
6462 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6463 {
6464 uint32_t u32Shadow;
6465 if (fWhat & CPUMCTX_EXTRN_CR0)
6466 {
6467 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6468 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6469 VMXLOCAL_BREAK_RC(rc);
6470 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6471 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6472 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6473 CPUMSetGuestCR0(pVCpu, u32Val);
6474 VMMRZCallRing3Enable(pVCpu);
6475 }
6476
6477 if (fWhat & CPUMCTX_EXTRN_CR4)
6478 {
6479 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6480 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6481 VMXLOCAL_BREAK_RC(rc);
6482 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6483 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6484 CPUMSetGuestCR4(pVCpu, u32Val);
6485 }
6486
6487 if (fWhat & CPUMCTX_EXTRN_CR3)
6488 {
6489 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6490 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6491 || ( pVM->hm.s.fNestedPaging
6492 && CPUMIsGuestPagingEnabledEx(pCtx)))
6493 {
6494 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6495 if (pCtx->cr3 != u64Val)
6496 {
6497 CPUMSetGuestCR3(pVCpu, u64Val);
6498 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6499 }
6500
6501 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6502 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6503 if (CPUMIsGuestInPAEModeEx(pCtx))
6504 {
6505 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6506 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6507 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6508 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6509 VMXLOCAL_BREAK_RC(rc);
6510 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6511 }
6512 }
6513 }
6514 }
6515 } while (0);
6516
6517 if (RT_SUCCESS(rc))
6518 {
6519 /* Update fExtrn. */
6520 pCtx->fExtrn &= ~fWhat;
6521
6522 /* If everything has been imported, clear the HM keeper bit. */
6523 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6524 {
6525 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6526 Assert(!pCtx->fExtrn);
6527 }
6528 }
6529 }
6530 else
6531 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6532
6533 ASMSetFlags(fEFlags);
6534
6535 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6536
6537 /*
6538 * Honor any pending CR3 updates.
6539 *
6540 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6541 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6542 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6543 *
6544 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6545 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6546 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6547 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6548 *
6549 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6550 */
6551 if (VMMRZCallRing3IsEnabled(pVCpu))
6552 {
6553 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6554 {
6555 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6556 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6557 }
6558
6559 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6560 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6561
6562 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6563 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6564 }
6565
6566 return VINF_SUCCESS;
6567#undef VMXLOCAL_BREAK_RC
6568}
6569
6570
6571/**
6572 * Saves the guest state from the VMCS into the guest-CPU context.
6573 *
6574 * @returns VBox status code.
6575 * @param pVCpu The cross context virtual CPU structure.
6576 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6577 */
6578VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6579{
6580 return hmR0VmxImportGuestState(pVCpu, fWhat);
6581}
6582
6583
6584/**
6585 * Check per-VM and per-VCPU force flag actions that require us to go back to
6586 * ring-3 for one reason or another.
6587 *
6588 * @returns Strict VBox status code (i.e. informational status codes too)
6589 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6590 * ring-3.
6591 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6592 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6593 * interrupts)
6594 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6595 * all EMTs to be in ring-3.
6596 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6597 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6598 * to the EM loop.
6599 *
6600 * @param pVCpu The cross context virtual CPU structure.
6601 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6602 * out-of-sync. Make sure to update the required fields
6603 * before using them.
6604 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6605 */
6606static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6607{
6608 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6609
6610 /*
6611 * Anything pending? Should be more likely than not if we're doing a good job.
6612 */
6613 PVM pVM = pVCpu->CTX_SUFF(pVM);
6614 if ( !fStepping
6615 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6616 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6617 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6618 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6619 return VINF_SUCCESS;
6620
6621 /* Pending PGM C3 sync. */
6622 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6623 {
6624 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6625 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6626 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6627 if (rcStrict2 != VINF_SUCCESS)
6628 {
6629 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6630 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6631 return rcStrict2;
6632 }
6633 }
6634
6635 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6636 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6637 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6638 {
6639 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6640 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6641 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6642 return rc2;
6643 }
6644
6645 /* Pending VM request packets, such as hardware interrupts. */
6646 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6647 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6648 {
6649 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6650 return VINF_EM_PENDING_REQUEST;
6651 }
6652
6653 /* Pending PGM pool flushes. */
6654 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6655 {
6656 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6657 return VINF_PGM_POOL_FLUSH_PENDING;
6658 }
6659
6660 /* Pending DMA requests. */
6661 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6662 {
6663 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6664 return VINF_EM_RAW_TO_R3;
6665 }
6666
6667 return VINF_SUCCESS;
6668}
6669
6670
6671/**
6672 * Converts any TRPM trap into a pending HM event. This is typically used when
6673 * entering from ring-3 (not longjmp returns).
6674 *
6675 * @param pVCpu The cross context virtual CPU structure.
6676 */
6677static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6678{
6679 Assert(TRPMHasTrap(pVCpu));
6680 Assert(!pVCpu->hm.s.Event.fPending);
6681
6682 uint8_t uVector;
6683 TRPMEVENT enmTrpmEvent;
6684 RTGCUINT uErrCode;
6685 RTGCUINTPTR GCPtrFaultAddress;
6686 uint8_t cbInstr;
6687
6688 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6689 AssertRC(rc);
6690
6691 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6692 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6693 if (enmTrpmEvent == TRPM_TRAP)
6694 {
6695 switch (uVector)
6696 {
6697 case X86_XCPT_NMI:
6698 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6699 break;
6700
6701 case X86_XCPT_BP:
6702 case X86_XCPT_OF:
6703 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6704 break;
6705
6706 case X86_XCPT_PF:
6707 case X86_XCPT_DF:
6708 case X86_XCPT_TS:
6709 case X86_XCPT_NP:
6710 case X86_XCPT_SS:
6711 case X86_XCPT_GP:
6712 case X86_XCPT_AC:
6713 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6714 RT_FALL_THRU();
6715 default:
6716 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6717 break;
6718 }
6719 }
6720 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6721 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6722 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6723 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6724 else
6725 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6726
6727 rc = TRPMResetTrap(pVCpu);
6728 AssertRC(rc);
6729 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6730 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6731
6732 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6733}
6734
6735
6736/**
6737 * Converts the pending HM event into a TRPM trap.
6738 *
6739 * @param pVCpu The cross context virtual CPU structure.
6740 */
6741static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6742{
6743 Assert(pVCpu->hm.s.Event.fPending);
6744
6745 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6746 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6747 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6748 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6749
6750 /* If a trap was already pending, we did something wrong! */
6751 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6752
6753 TRPMEVENT enmTrapType;
6754 switch (uVectorType)
6755 {
6756 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6757 enmTrapType = TRPM_HARDWARE_INT;
6758 break;
6759
6760 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6761 enmTrapType = TRPM_SOFTWARE_INT;
6762 break;
6763
6764 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6765 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6766 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6767 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6768 enmTrapType = TRPM_TRAP;
6769 break;
6770
6771 default:
6772 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6773 enmTrapType = TRPM_32BIT_HACK;
6774 break;
6775 }
6776
6777 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6778
6779 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6780 AssertRC(rc);
6781
6782 if (fErrorCodeValid)
6783 TRPMSetErrorCode(pVCpu, uErrorCode);
6784
6785 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6786 && uVector == X86_XCPT_PF)
6787 {
6788 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6789 }
6790 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6791 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6792 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6793 {
6794 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6795 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6796 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6797 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6798 }
6799
6800 /* Clear any pending events from the VMCS. */
6801 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6802 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6803
6804 /* We're now done converting the pending event. */
6805 pVCpu->hm.s.Event.fPending = false;
6806}
6807
6808
6809/**
6810 * Does the necessary state syncing before returning to ring-3 for any reason
6811 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6812 *
6813 * @returns VBox status code.
6814 * @param pVCpu The cross context virtual CPU structure.
6815 * @param fImportState Whether to import the guest state from the VMCS back
6816 * to the guest-CPU context.
6817 *
6818 * @remarks No-long-jmp zone!!!
6819 */
6820static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6821{
6822 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6823 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6824
6825 RTCPUID idCpu = RTMpCpuId();
6826 Log4Func(("HostCpuId=%u\n", idCpu));
6827
6828 /*
6829 * !!! IMPORTANT !!!
6830 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6831 */
6832
6833 /* Save the guest state if necessary. */
6834 if (fImportState)
6835 {
6836 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6837 AssertRCReturn(rc, rc);
6838 }
6839
6840 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6841 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6842 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6843
6844 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6845#ifdef VBOX_STRICT
6846 if (CPUMIsHyperDebugStateActive(pVCpu))
6847 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6848#endif
6849 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6850 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6851 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6852
6853#if HC_ARCH_BITS == 64
6854 /* Restore host-state bits that VT-x only restores partially. */
6855 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6856 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6857 {
6858 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6859 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6860 }
6861 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6862#endif
6863
6864 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6865 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6866 {
6867 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6868 if (!fImportState)
6869 {
6870 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6871 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6872 AssertRCReturn(rc, rc);
6873 }
6874 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6875 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6876 }
6877 else
6878 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6879
6880 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6881 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6882
6883 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6884 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6885 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6886 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6887 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6888 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6889 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6890 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6891 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6892
6893 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6894
6895 /** @todo This partially defeats the purpose of having preemption hooks.
6896 * The problem is, deregistering the hooks should be moved to a place that
6897 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6898 * context.
6899 */
6900 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6901 {
6902 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6903 AssertRCReturn(rc, rc);
6904
6905 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6906 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6907 }
6908 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6909 NOREF(idCpu);
6910
6911 return VINF_SUCCESS;
6912}
6913
6914
6915/**
6916 * Leaves the VT-x session.
6917 *
6918 * @returns VBox status code.
6919 * @param pVCpu The cross context virtual CPU structure.
6920 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6921 * out-of-sync. Make sure to update the required fields
6922 * before using them.
6923 *
6924 * @remarks No-long-jmp zone!!!
6925 */
6926static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6927{
6928 HM_DISABLE_PREEMPT();
6929 HMVMX_ASSERT_CPU_SAFE();
6930 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6931 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6932
6933 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6934 and done this from the VMXR0ThreadCtxCallback(). */
6935 if (!pVCpu->hm.s.fLeaveDone)
6936 {
6937 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6938 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6939 pVCpu->hm.s.fLeaveDone = true;
6940 }
6941 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6942
6943 /*
6944 * !!! IMPORTANT !!!
6945 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6946 */
6947
6948 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6949 /** @todo Deregistering here means we need to VMCLEAR always
6950 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6951 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6952 VMMR0ThreadCtxHookDisable(pVCpu);
6953
6954 /* Leave HM context. This takes care of local init (term). */
6955 int rc = HMR0LeaveCpu(pVCpu);
6956
6957 HM_RESTORE_PREEMPT();
6958 return rc;
6959}
6960
6961
6962/**
6963 * Does the necessary state syncing before doing a longjmp to ring-3.
6964 *
6965 * @returns VBox status code.
6966 * @param pVCpu The cross context virtual CPU structure.
6967 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6968 * out-of-sync. Make sure to update the required fields
6969 * before using them.
6970 *
6971 * @remarks No-long-jmp zone!!!
6972 */
6973DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6974{
6975 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6976}
6977
6978
6979/**
6980 * Take necessary actions before going back to ring-3.
6981 *
6982 * An action requires us to go back to ring-3. This function does the necessary
6983 * steps before we can safely return to ring-3. This is not the same as longjmps
6984 * to ring-3, this is voluntary and prepares the guest so it may continue
6985 * executing outside HM (recompiler/IEM).
6986 *
6987 * @returns VBox status code.
6988 * @param pVCpu The cross context virtual CPU structure.
6989 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6990 * out-of-sync. Make sure to update the required fields
6991 * before using them.
6992 * @param rcExit The reason for exiting to ring-3. Can be
6993 * VINF_VMM_UNKNOWN_RING3_CALL.
6994 */
6995static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6996{
6997 Assert(pVCpu);
6998 Assert(pMixedCtx);
6999 HMVMX_ASSERT_PREEMPT_SAFE();
7000
7001 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7002 {
7003 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7004 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7005 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7006 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7007 }
7008
7009 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7010 VMMRZCallRing3Disable(pVCpu);
7011 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7012
7013 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7014 if (pVCpu->hm.s.Event.fPending)
7015 {
7016 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7017 Assert(!pVCpu->hm.s.Event.fPending);
7018 }
7019
7020 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7021 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7022
7023 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7024 and if we're injecting an event we should have a TRPM trap pending. */
7025 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7026#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7027 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7028#endif
7029
7030 /* Save guest state and restore host state bits. */
7031 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7032 AssertRCReturn(rc, rc);
7033 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7034 /* Thread-context hooks are unregistered at this point!!! */
7035
7036 /* Sync recompiler state. */
7037 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7038 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7039 | CPUM_CHANGED_LDTR
7040 | CPUM_CHANGED_GDTR
7041 | CPUM_CHANGED_IDTR
7042 | CPUM_CHANGED_TR
7043 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7044 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7045 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7046 {
7047 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7048 }
7049
7050 Assert(!pVCpu->hm.s.fClearTrapFlag);
7051
7052 /* Update the exit-to-ring 3 reason. */
7053 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7054
7055 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7056 if (rcExit != VINF_EM_RAW_INTERRUPT)
7057 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7058
7059 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7060
7061 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7062 VMMRZCallRing3RemoveNotification(pVCpu);
7063 VMMRZCallRing3Enable(pVCpu);
7064
7065 return rc;
7066}
7067
7068
7069/**
7070 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7071 * longjump to ring-3 and possibly get preempted.
7072 *
7073 * @returns VBox status code.
7074 * @param pVCpu The cross context virtual CPU structure.
7075 * @param enmOperation The operation causing the ring-3 longjump.
7076 * @param pvUser Opaque pointer to the guest-CPU context. The data
7077 * may be out-of-sync. Make sure to update the required
7078 * fields before using them.
7079 */
7080static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7081{
7082 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7083 {
7084 /*
7085 * !!! IMPORTANT !!!
7086 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7087 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7088 */
7089 VMMRZCallRing3RemoveNotification(pVCpu);
7090 VMMRZCallRing3Disable(pVCpu);
7091 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7092 RTThreadPreemptDisable(&PreemptState);
7093
7094 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7095 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7096
7097#if HC_ARCH_BITS == 64
7098 /* Restore host-state bits that VT-x only restores partially. */
7099 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7100 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7101 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7102 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7103#endif
7104
7105 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7106 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7107 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7108
7109 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7110 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7111 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7112 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7113 {
7114 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7115 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7116 }
7117
7118 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7119 VMMR0ThreadCtxHookDisable(pVCpu);
7120 HMR0LeaveCpu(pVCpu);
7121 RTThreadPreemptRestore(&PreemptState);
7122 return VINF_SUCCESS;
7123 }
7124
7125 Assert(pVCpu);
7126 Assert(pvUser);
7127 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7128 HMVMX_ASSERT_PREEMPT_SAFE();
7129
7130 VMMRZCallRing3Disable(pVCpu);
7131 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7132
7133 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7134
7135 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7136 AssertRCReturn(rc, rc);
7137
7138 VMMRZCallRing3Enable(pVCpu);
7139 return VINF_SUCCESS;
7140}
7141
7142
7143/**
7144 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7145 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7146 *
7147 * @param pVCpu The cross context virtual CPU structure.
7148 */
7149DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7150{
7151 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7152 {
7153 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7154 {
7155 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7156 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7157 AssertRC(rc);
7158 Log4Func(("Setup interrupt-window exiting\n"));
7159 }
7160 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7161}
7162
7163
7164/**
7165 * Clears the interrupt-window exiting control in the VMCS.
7166 *
7167 * @param pVCpu The cross context virtual CPU structure.
7168 */
7169DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7170{
7171 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7172 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7173 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7174 AssertRC(rc);
7175 Log4Func(("Cleared interrupt-window exiting\n"));
7176}
7177
7178
7179/**
7180 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7181 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7182 *
7183 * @param pVCpu The cross context virtual CPU structure.
7184 */
7185DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7186{
7187 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7188 {
7189 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7190 {
7191 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7192 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7193 AssertRC(rc);
7194 Log4Func(("Setup NMI-window exiting\n"));
7195 }
7196 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7197}
7198
7199
7200/**
7201 * Clears the NMI-window exiting control in the VMCS.
7202 *
7203 * @param pVCpu The cross context virtual CPU structure.
7204 */
7205DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7206{
7207 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7208 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7209 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7210 AssertRC(rc);
7211 Log4Func(("Cleared NMI-window exiting\n"));
7212}
7213
7214
7215/**
7216 * Evaluates the event to be delivered to the guest and sets it as the pending
7217 * event.
7218 *
7219 * @returns The VT-x guest-interruptibility state.
7220 * @param pVCpu The cross context virtual CPU structure.
7221 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7222 * out-of-sync. Make sure to update the required fields
7223 * before using them.
7224 */
7225static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7226{
7227 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7228 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7229 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7230 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7231 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7232
7233 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7234 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7235 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7236 Assert(!TRPMHasTrap(pVCpu));
7237
7238 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7239 APICUpdatePendingInterrupts(pVCpu);
7240
7241 /*
7242 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7243 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7244 */
7245 /** @todo SMI. SMIs take priority over NMIs. */
7246 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7247 {
7248 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7249 if ( !pVCpu->hm.s.Event.fPending
7250 && !fBlockNmi
7251 && !fBlockSti
7252 && !fBlockMovSS)
7253 {
7254 Log4Func(("Pending NMI\n"));
7255 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7256 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7257
7258 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7259 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7260 }
7261 else
7262 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7263 }
7264 /*
7265 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7266 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7267 */
7268 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7269 && !pVCpu->hm.s.fSingleInstruction)
7270 {
7271 Assert(!DBGFIsStepping(pVCpu));
7272 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7273 AssertRCReturn(rc, 0);
7274 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7275 if ( !pVCpu->hm.s.Event.fPending
7276 && !fBlockInt
7277 && !fBlockSti
7278 && !fBlockMovSS)
7279 {
7280 uint8_t u8Interrupt;
7281 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7282 if (RT_SUCCESS(rc))
7283 {
7284 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7285 uint32_t u32IntInfo = u8Interrupt
7286 | VMX_EXIT_INTERRUPTION_INFO_VALID
7287 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7288
7289 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7290 }
7291 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7292 {
7293 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7294 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7295 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7296
7297 /*
7298 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7299 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7300 * need to re-set this force-flag here.
7301 */
7302 }
7303 else
7304 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7305 }
7306 else
7307 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7308 }
7309
7310 return fIntrState;
7311}
7312
7313
7314/**
7315 * Sets a pending-debug exception to be delivered to the guest if the guest is
7316 * single-stepping in the VMCS.
7317 *
7318 * @param pVCpu The cross context virtual CPU structure.
7319 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7320 * out-of-sync. Make sure to update the required fields
7321 * before using them.
7322 */
7323DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7324{
7325 RT_NOREF(pVCpu);
7326 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7327 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7328}
7329
7330
7331/**
7332 * Injects any pending events into the guest if the guest is in a state to
7333 * receive them.
7334 *
7335 * @returns Strict VBox status code (i.e. informational status codes too).
7336 * @param pVCpu The cross context virtual CPU structure.
7337 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7338 * out-of-sync. Make sure to update the required fields
7339 * before using them.
7340 * @param fIntrState The VT-x guest-interruptibility state.
7341 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7342 * return VINF_EM_DBG_STEPPED if the event was
7343 * dispatched directly.
7344 */
7345static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7346{
7347 HMVMX_ASSERT_PREEMPT_SAFE();
7348 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7349
7350 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7351 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7352
7353 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7354 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7355 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7356 Assert(!TRPMHasTrap(pVCpu));
7357
7358 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7359 if (pVCpu->hm.s.Event.fPending)
7360 {
7361 /*
7362 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7363 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7364 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7365 *
7366 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7367 */
7368 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7369#ifdef VBOX_STRICT
7370 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7371 {
7372 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7373 Assert(!fBlockInt);
7374 Assert(!fBlockSti);
7375 Assert(!fBlockMovSS);
7376 }
7377 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7378 {
7379 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7380 Assert(!fBlockSti);
7381 Assert(!fBlockMovSS);
7382 Assert(!fBlockNmi);
7383 }
7384#endif
7385 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7386 uIntType));
7387 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7388 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7389 &fIntrState);
7390 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7391
7392 /* Update the interruptibility-state as it could have been changed by
7393 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7394 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7395 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7396
7397 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7398 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7399 else
7400 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7401 }
7402
7403 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7404 if ( fBlockSti
7405 || fBlockMovSS)
7406 {
7407 if (!pVCpu->hm.s.fSingleInstruction)
7408 {
7409 /*
7410 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7411 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7412 * See Intel spec. 27.3.4 "Saving Non-Register State".
7413 */
7414 Assert(!DBGFIsStepping(pVCpu));
7415 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7416 AssertRCReturn(rc, rc);
7417 if (pMixedCtx->eflags.Bits.u1TF)
7418 {
7419 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7420 AssertRCReturn(rc2, rc2);
7421 }
7422 }
7423 else if (pMixedCtx->eflags.Bits.u1TF)
7424 {
7425 /*
7426 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7427 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7428 */
7429 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7430 fIntrState = 0;
7431 }
7432 }
7433
7434 /*
7435 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7436 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7437 */
7438 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7439 AssertRCReturn(rc3, rc3);
7440
7441 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7442 NOREF(fBlockMovSS); NOREF(fBlockSti);
7443 return rcStrict;
7444}
7445
7446
7447/**
7448 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7449 *
7450 * @param pVCpu The cross context virtual CPU structure.
7451 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7452 * out-of-sync. Make sure to update the required fields
7453 * before using them.
7454 */
7455DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7456{
7457 NOREF(pMixedCtx);
7458 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7459 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7460}
7461
7462
7463/**
7464 * Injects a double-fault (\#DF) exception into the VM.
7465 *
7466 * @returns Strict VBox status code (i.e. informational status codes too).
7467 * @param pVCpu The cross context virtual CPU structure.
7468 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7469 * out-of-sync. Make sure to update the required fields
7470 * before using them.
7471 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7472 * and should return VINF_EM_DBG_STEPPED if the event
7473 * is injected directly (register modified by us, not
7474 * by hardware on VM-entry).
7475 * @param pfIntrState Pointer to the current guest interruptibility-state.
7476 * This interruptibility-state will be updated if
7477 * necessary. This cannot not be NULL.
7478 */
7479DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7480{
7481 NOREF(pMixedCtx);
7482 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7483 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7484 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7485 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7486 pfIntrState);
7487}
7488
7489
7490/**
7491 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7492 *
7493 * @param pVCpu The cross context virtual CPU structure.
7494 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7495 * out-of-sync. Make sure to update the required fields
7496 * before using them.
7497 */
7498DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7499{
7500 NOREF(pMixedCtx);
7501 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7502 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7503 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7504}
7505
7506
7507/**
7508 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7509 *
7510 * @param pVCpu The cross context virtual CPU structure.
7511 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7512 * out-of-sync. Make sure to update the required fields
7513 * before using them.
7514 * @param cbInstr The value of RIP that is to be pushed on the guest
7515 * stack.
7516 */
7517DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7518{
7519 NOREF(pMixedCtx);
7520 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7521 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7522 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7523}
7524
7525
7526/**
7527 * Injects a general-protection (\#GP) fault into the VM.
7528 *
7529 * @returns Strict VBox status code (i.e. informational status codes too).
7530 * @param pVCpu The cross context virtual CPU structure.
7531 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7532 * out-of-sync. Make sure to update the required fields
7533 * before using them.
7534 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7535 * mode, i.e. in real-mode it's not valid).
7536 * @param u32ErrorCode The error code associated with the \#GP.
7537 * @param fStepping Whether we're running in
7538 * hmR0VmxRunGuestCodeStep() and should return
7539 * VINF_EM_DBG_STEPPED if the event is injected
7540 * directly (register modified by us, not by
7541 * hardware on VM-entry).
7542 * @param pfIntrState Pointer to the current guest interruptibility-state.
7543 * This interruptibility-state will be updated if
7544 * necessary. This cannot not be NULL.
7545 */
7546DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7547 bool fStepping, uint32_t *pfIntrState)
7548{
7549 NOREF(pMixedCtx);
7550 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7551 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7552 if (fErrorCodeValid)
7553 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7554 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7555 pfIntrState);
7556}
7557
7558
7559#if 0 /* unused */
7560/**
7561 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7562 * VM.
7563 *
7564 * @param pVCpu The cross context virtual CPU structure.
7565 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7566 * out-of-sync. Make sure to update the required fields
7567 * before using them.
7568 * @param u32ErrorCode The error code associated with the \#GP.
7569 */
7570DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7571{
7572 NOREF(pMixedCtx);
7573 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7574 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7575 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7576 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7577}
7578#endif /* unused */
7579
7580
7581/**
7582 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7583 *
7584 * @param pVCpu The cross context virtual CPU structure.
7585 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7586 * out-of-sync. Make sure to update the required fields
7587 * before using them.
7588 * @param uVector The software interrupt vector number.
7589 * @param cbInstr The value of RIP that is to be pushed on the guest
7590 * stack.
7591 */
7592DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7593{
7594 NOREF(pMixedCtx);
7595 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7596 if ( uVector == X86_XCPT_BP
7597 || uVector == X86_XCPT_OF)
7598 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7599 else
7600 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7601 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7602}
7603
7604
7605/**
7606 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7607 * stack.
7608 *
7609 * @returns Strict VBox status code (i.e. informational status codes too).
7610 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7611 * @param pVM The cross context VM structure.
7612 * @param pMixedCtx Pointer to the guest-CPU context.
7613 * @param uValue The value to push to the guest stack.
7614 */
7615DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7616{
7617 /*
7618 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7619 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7620 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7621 */
7622 if (pMixedCtx->sp == 1)
7623 return VINF_EM_RESET;
7624 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7625 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7626 AssertRC(rc);
7627 return rc;
7628}
7629
7630
7631/**
7632 * Injects an event into the guest upon VM-entry by updating the relevant fields
7633 * in the VM-entry area in the VMCS.
7634 *
7635 * @returns Strict VBox status code (i.e. informational status codes too).
7636 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7637 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7638 *
7639 * @param pVCpu The cross context virtual CPU structure.
7640 * @param u64IntInfo The VM-entry interruption-information field.
7641 * @param cbInstr The VM-entry instruction length in bytes (for
7642 * software interrupts, exceptions and privileged
7643 * software exceptions).
7644 * @param u32ErrCode The VM-entry exception error code.
7645 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7646 * @param pfIntrState Pointer to the current guest interruptibility-state.
7647 * This interruptibility-state will be updated if
7648 * necessary. This cannot not be NULL.
7649 * @param fStepping Whether we're running in
7650 * hmR0VmxRunGuestCodeStep() and should return
7651 * VINF_EM_DBG_STEPPED if the event is injected
7652 * directly (register modified by us, not by
7653 * hardware on VM-entry).
7654 */
7655static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7656 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7657{
7658 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7659 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7660 Assert(pfIntrState);
7661
7662 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7663 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7664 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7665 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7666
7667#ifdef VBOX_STRICT
7668 /*
7669 * Validate the error-code-valid bit for hardware exceptions.
7670 * No error codes for exceptions in real-mode.
7671 *
7672 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7673 */
7674 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7675 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7676 {
7677 switch (uVector)
7678 {
7679 case X86_XCPT_PF:
7680 case X86_XCPT_DF:
7681 case X86_XCPT_TS:
7682 case X86_XCPT_NP:
7683 case X86_XCPT_SS:
7684 case X86_XCPT_GP:
7685 case X86_XCPT_AC:
7686 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7687 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7688 RT_FALL_THRU();
7689 default:
7690 break;
7691 }
7692 }
7693#endif
7694
7695 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7696 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7697 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7698
7699 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7700
7701 /*
7702 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7703 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7704 * interrupt handler in the (real-mode) guest.
7705 *
7706 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7707 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7708 */
7709 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7710 {
7711 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7712 {
7713 /*
7714 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7715 * set the deliver-error-code bit.
7716 *
7717 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7718 */
7719 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7720 }
7721 else
7722 {
7723 PVM pVM = pVCpu->CTX_SUFF(pVM);
7724 Assert(PDMVmmDevHeapIsEnabled(pVM));
7725 Assert(pVM->hm.s.vmx.pRealModeTSS);
7726
7727 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7728 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7729 | CPUMCTX_EXTRN_TABLE_MASK
7730 | CPUMCTX_EXTRN_RIP
7731 | CPUMCTX_EXTRN_RSP
7732 | CPUMCTX_EXTRN_RFLAGS);
7733 AssertRCReturn(rc2, rc2);
7734
7735 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7736 size_t const cbIdtEntry = sizeof(X86IDTR16);
7737 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7738 {
7739 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7740 if (uVector == X86_XCPT_DF)
7741 return VINF_EM_RESET;
7742
7743 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7744 if (uVector == X86_XCPT_GP)
7745 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7746
7747 /*
7748 * If we're injecting an event with no valid IDT entry, inject a #GP.
7749 * No error codes for exceptions in real-mode.
7750 *
7751 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7752 */
7753 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7754 pfIntrState);
7755 }
7756
7757 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7758 uint16_t uGuestIp = pMixedCtx->ip;
7759 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7760 {
7761 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7762 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7763 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7764 }
7765 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7766 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7767
7768 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7769 X86IDTR16 IdtEntry;
7770 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7771 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7772 AssertRCReturn(rc2, rc2);
7773
7774 /* Construct the stack frame for the interrupt/exception handler. */
7775 VBOXSTRICTRC rcStrict;
7776 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7777 if (rcStrict == VINF_SUCCESS)
7778 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7779 if (rcStrict == VINF_SUCCESS)
7780 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7781
7782 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7783 if (rcStrict == VINF_SUCCESS)
7784 {
7785 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7786 pMixedCtx->rip = IdtEntry.offSel;
7787 pMixedCtx->cs.Sel = IdtEntry.uSel;
7788 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7789 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7790 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7791 && uVector == X86_XCPT_PF)
7792 pMixedCtx->cr2 = GCPtrFaultAddress;
7793
7794 /* If any other guest-state bits are changed here, make sure to update
7795 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7796 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7797 | HM_CHANGED_GUEST_CR2
7798 | HM_CHANGED_GUEST_RIP
7799 | HM_CHANGED_GUEST_RFLAGS
7800 | HM_CHANGED_GUEST_RSP);
7801
7802 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7803 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7804 {
7805 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7806 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7807 Log4Func(("Clearing inhibition due to STI\n"));
7808 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7809 }
7810 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7811 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7812
7813 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7814 it, if we are returning to ring-3 before executing guest code. */
7815 pVCpu->hm.s.Event.fPending = false;
7816
7817 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7818 if (fStepping)
7819 rcStrict = VINF_EM_DBG_STEPPED;
7820 }
7821 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7822 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7823 return rcStrict;
7824 }
7825 }
7826
7827 /* Validate. */
7828 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7829 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7830 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7831
7832 /* Inject. */
7833 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7834 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7835 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7836 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7837 AssertRCReturn(rc, rc);
7838
7839 /* Update CR2. */
7840 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7841 && uVector == X86_XCPT_PF)
7842 pMixedCtx->cr2 = GCPtrFaultAddress;
7843
7844 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7845
7846 return VINF_SUCCESS;
7847}
7848
7849
7850/**
7851 * Clears the interrupt-window exiting control in the VMCS and if necessary
7852 * clears the current event in the VMCS as well.
7853 *
7854 * @returns VBox status code.
7855 * @param pVCpu The cross context virtual CPU structure.
7856 *
7857 * @remarks Use this function only to clear events that have not yet been
7858 * delivered to the guest but are injected in the VMCS!
7859 * @remarks No-long-jump zone!!!
7860 */
7861static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7862{
7863 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7864 {
7865 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7866 Log4Func(("Cleared interrupt widow\n"));
7867 }
7868
7869 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7870 {
7871 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7872 Log4Func(("Cleared interrupt widow\n"));
7873 }
7874}
7875
7876
7877/**
7878 * Enters the VT-x session.
7879 *
7880 * @returns VBox status code.
7881 * @param pVCpu The cross context virtual CPU structure.
7882 * @param pHostCpu Pointer to the global CPU info struct.
7883 */
7884VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7885{
7886 AssertPtr(pVCpu);
7887 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7888 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7889 RT_NOREF(pHostCpu);
7890
7891 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7892 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7893 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7894
7895#ifdef VBOX_STRICT
7896 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7897 RTCCUINTREG uHostCR4 = ASMGetCR4();
7898 if (!(uHostCR4 & X86_CR4_VMXE))
7899 {
7900 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7901 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7902 }
7903#endif
7904
7905 /*
7906 * Load the VCPU's VMCS as the current (and active) one.
7907 */
7908 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7909 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7910 if (RT_FAILURE(rc))
7911 return rc;
7912
7913 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7914 pVCpu->hm.s.fLeaveDone = false;
7915 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7916
7917 return VINF_SUCCESS;
7918}
7919
7920
7921/**
7922 * The thread-context callback (only on platforms which support it).
7923 *
7924 * @param enmEvent The thread-context event.
7925 * @param pVCpu The cross context virtual CPU structure.
7926 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7927 * @thread EMT(pVCpu)
7928 */
7929VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7930{
7931 NOREF(fGlobalInit);
7932
7933 switch (enmEvent)
7934 {
7935 case RTTHREADCTXEVENT_OUT:
7936 {
7937 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7938 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7939 VMCPU_ASSERT_EMT(pVCpu);
7940
7941 /* No longjmps (logger flushes, locks) in this fragile context. */
7942 VMMRZCallRing3Disable(pVCpu);
7943 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7944
7945 /*
7946 * Restore host-state (FPU, debug etc.)
7947 */
7948 if (!pVCpu->hm.s.fLeaveDone)
7949 {
7950 /*
7951 * Do -not- import the guest-state here as we might already be in the middle of importing
7952 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7953 */
7954 hmR0VmxLeave(pVCpu, false /* fImportState */);
7955 pVCpu->hm.s.fLeaveDone = true;
7956 }
7957
7958 /* Leave HM context, takes care of local init (term). */
7959 int rc = HMR0LeaveCpu(pVCpu);
7960 AssertRC(rc); NOREF(rc);
7961
7962 /* Restore longjmp state. */
7963 VMMRZCallRing3Enable(pVCpu);
7964 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7965 break;
7966 }
7967
7968 case RTTHREADCTXEVENT_IN:
7969 {
7970 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7971 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7972 VMCPU_ASSERT_EMT(pVCpu);
7973
7974 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7975 VMMRZCallRing3Disable(pVCpu);
7976 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7977
7978 /* Initialize the bare minimum state required for HM. This takes care of
7979 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7980 int rc = hmR0EnterCpu(pVCpu);
7981 AssertRC(rc);
7982 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7983 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7984
7985 /* Load the active VMCS as the current one. */
7986 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7987 {
7988 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7989 AssertRC(rc); NOREF(rc);
7990 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7991 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7992 }
7993 pVCpu->hm.s.fLeaveDone = false;
7994
7995 /* Restore longjmp state. */
7996 VMMRZCallRing3Enable(pVCpu);
7997 break;
7998 }
7999
8000 default:
8001 break;
8002 }
8003}
8004
8005
8006/**
8007 * Exports the host state into the VMCS host-state area.
8008 * Sets up the VM-exit MSR-load area.
8009 *
8010 * The CPU state will be loaded from these fields on every successful VM-exit.
8011 *
8012 * @returns VBox status code.
8013 * @param pVCpu The cross context virtual CPU structure.
8014 *
8015 * @remarks No-long-jump zone!!!
8016 */
8017static int hmR0VmxExportHostState(PVMCPU pVCpu)
8018{
8019 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8020
8021 int rc = VINF_SUCCESS;
8022 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8023 {
8024 rc = hmR0VmxExportHostControlRegs();
8025 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8026
8027 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8028 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8029
8030 rc = hmR0VmxExportHostMsrs(pVCpu);
8031 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8032
8033 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8034 }
8035 return rc;
8036}
8037
8038
8039/**
8040 * Saves the host state in the VMCS host-state.
8041 *
8042 * @returns VBox status code.
8043 * @param pVCpu The cross context virtual CPU structure.
8044 *
8045 * @remarks No-long-jump zone!!!
8046 */
8047VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8048{
8049 AssertPtr(pVCpu);
8050 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8051
8052 /*
8053 * Export the host state here while entering HM context.
8054 * When thread-context hooks are used, we might get preempted and have to re-save the host
8055 * state but most of the time we won't be, so do it here before we disable interrupts.
8056 */
8057 return hmR0VmxExportHostState(pVCpu);
8058}
8059
8060
8061/**
8062 * Exports the guest state into the VMCS guest-state area.
8063 *
8064 * The will typically be done before VM-entry when the guest-CPU state and the
8065 * VMCS state may potentially be out of sync.
8066 *
8067 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8068 * VM-entry controls.
8069 * Sets up the appropriate VMX non-root function to execute guest code based on
8070 * the guest CPU mode.
8071 *
8072 * @returns VBox strict status code.
8073 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8074 * without unrestricted guest access and the VMMDev is not presently
8075 * mapped (e.g. EFI32).
8076 *
8077 * @param pVCpu The cross context virtual CPU structure.
8078 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8079 * out-of-sync. Make sure to update the required fields
8080 * before using them.
8081 *
8082 * @remarks No-long-jump zone!!!
8083 */
8084static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8085{
8086 AssertPtr(pVCpu);
8087 AssertPtr(pMixedCtx);
8088 HMVMX_ASSERT_PREEMPT_SAFE();
8089
8090 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8091
8092 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8093
8094 /* Determine real-on-v86 mode. */
8095 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8096 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8097 && CPUMIsGuestInRealModeEx(pMixedCtx))
8098 {
8099 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8100 }
8101
8102 /*
8103 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8104 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8105 */
8106 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8107 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8108
8109 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8110 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8111 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8112
8113 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8114 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8115 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8116
8117 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8118 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8119
8120 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8121 if (rcStrict == VINF_SUCCESS)
8122 { /* likely */ }
8123 else
8124 {
8125 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8126 return rcStrict;
8127 }
8128
8129 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8130 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8131
8132 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8133 may alter controls if we determine we don't have to swap EFER after all. */
8134 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8135 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8136
8137 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8138 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8139
8140 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8141 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8142 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8143
8144 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8145 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8146 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8147 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8148 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8149 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8150
8151 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8152 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8153 | HM_CHANGED_GUEST_CR2
8154 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8155 | HM_CHANGED_GUEST_X87
8156 | HM_CHANGED_GUEST_SSE_AVX
8157 | HM_CHANGED_GUEST_OTHER_XSAVE
8158 | HM_CHANGED_GUEST_XCRx
8159 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8160 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8161 | HM_CHANGED_GUEST_TSC_AUX
8162 | HM_CHANGED_GUEST_OTHER_MSRS
8163 | HM_CHANGED_GUEST_HWVIRT
8164 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8165
8166 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8167 return rc;
8168}
8169
8170
8171/**
8172 * Exports the state shared between the host and guest into the VMCS.
8173 *
8174 * @param pVCpu The cross context virtual CPU structure.
8175 * @param pCtx Pointer to the guest-CPU context.
8176 *
8177 * @remarks No-long-jump zone!!!
8178 */
8179static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8180{
8181 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8182 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8183
8184 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8185 {
8186 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8187 AssertRC(rc);
8188 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8189
8190 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8191 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8192 {
8193 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8194 AssertRC(rc);
8195 }
8196 }
8197
8198 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8199 {
8200 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8201 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8202 }
8203
8204 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8205 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8206}
8207
8208
8209/**
8210 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8211 *
8212 * @returns Strict VBox status code (i.e. informational status codes too).
8213 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8214 * without unrestricted guest access and the VMMDev is not presently
8215 * mapped (e.g. EFI32).
8216 *
8217 * @param pVCpu The cross context virtual CPU structure.
8218 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8219 * out-of-sync. Make sure to update the required fields
8220 * before using them.
8221 *
8222 * @remarks No-long-jump zone!!!
8223 */
8224static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8225{
8226 HMVMX_ASSERT_PREEMPT_SAFE();
8227 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8228 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8229
8230#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8231 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8232#endif
8233
8234 /*
8235 * For many exits it's only RIP that changes and hence try to export it first
8236 * without going through a lot of change flag checks.
8237 */
8238 VBOXSTRICTRC rcStrict;
8239 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8240 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8241 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8242 {
8243 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8244 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8245 { /* likely */}
8246 else
8247 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8248 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8249 }
8250 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8251 {
8252 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8253 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8254 { /* likely */}
8255 else
8256 {
8257 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8258 VBOXSTRICTRC_VAL(rcStrict)));
8259 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8260 return rcStrict;
8261 }
8262 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8263 }
8264 else
8265 rcStrict = VINF_SUCCESS;
8266
8267#ifdef VBOX_STRICT
8268 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8269 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8270 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8271 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8272 ("fCtxChanged=%#RX64\n", fCtxChanged));
8273#endif
8274 return rcStrict;
8275}
8276
8277
8278/**
8279 * Does the preparations before executing guest code in VT-x.
8280 *
8281 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8282 * recompiler/IEM. We must be cautious what we do here regarding committing
8283 * guest-state information into the VMCS assuming we assuredly execute the
8284 * guest in VT-x mode.
8285 *
8286 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8287 * the common-state (TRPM/forceflags), we must undo those changes so that the
8288 * recompiler/IEM can (and should) use them when it resumes guest execution.
8289 * Otherwise such operations must be done when we can no longer exit to ring-3.
8290 *
8291 * @returns Strict VBox status code (i.e. informational status codes too).
8292 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8293 * have been disabled.
8294 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8295 * double-fault into the guest.
8296 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8297 * dispatched directly.
8298 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8299 *
8300 * @param pVCpu The cross context virtual CPU structure.
8301 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8302 * out-of-sync. Make sure to update the required fields
8303 * before using them.
8304 * @param pVmxTransient Pointer to the VMX transient structure.
8305 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8306 * us ignore some of the reasons for returning to
8307 * ring-3, and return VINF_EM_DBG_STEPPED if event
8308 * dispatching took place.
8309 */
8310static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8311{
8312 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8313
8314#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8315 PGMRZDynMapFlushAutoSet(pVCpu);
8316#endif
8317
8318 /* Check force flag actions that might require us to go back to ring-3. */
8319 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8320 if (rcStrict == VINF_SUCCESS)
8321 { /* FFs doesn't get set all the time. */ }
8322 else
8323 return rcStrict;
8324
8325 /*
8326 * Setup the virtualized-APIC accesses.
8327 *
8328 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8329 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8330 *
8331 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8332 */
8333 PVM pVM = pVCpu->CTX_SUFF(pVM);
8334 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8335 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8336 && PDMHasApic(pVM))
8337 {
8338 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8339 Assert(u64MsrApicBase);
8340 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8341
8342 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8343
8344 /* Unalias any existing mapping. */
8345 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8346 AssertRCReturn(rc, rc);
8347
8348 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8349 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8350 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8351 AssertRCReturn(rc, rc);
8352
8353 /* Update the per-VCPU cache of the APIC base MSR. */
8354 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8355 }
8356
8357 if (TRPMHasTrap(pVCpu))
8358 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8359 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8360
8361 /*
8362 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8363 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8364 * also result in triple-faulting the VM.
8365 */
8366 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8367 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8368 { /* likely */ }
8369 else
8370 {
8371 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8372 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8373 return rcStrict;
8374 }
8375
8376 /*
8377 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8378 * import CR3 themselves. We will need to update them here as even as late as the above
8379 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8380 * the below force flags to be set.
8381 */
8382 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8383 {
8384 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8385 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8386 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8387 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8388 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8389 }
8390 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8391 {
8392 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8393 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8394 }
8395
8396 /*
8397 * No longjmps to ring-3 from this point on!!!
8398 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8399 * This also disables flushing of the R0-logger instance (if any).
8400 */
8401 VMMRZCallRing3Disable(pVCpu);
8402
8403 /*
8404 * Export the guest state bits.
8405 *
8406 * We cannot perform longjmps while loading the guest state because we do not preserve the
8407 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8408 * CPU migration.
8409 *
8410 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8411 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8412 * Hence, loading of the guest state needs to be done -after- injection of events.
8413 */
8414 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8415 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8416 { /* likely */ }
8417 else
8418 {
8419 VMMRZCallRing3Enable(pVCpu);
8420 return rcStrict;
8421 }
8422
8423 /*
8424 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8425 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8426 * preemption disabled for a while. Since this is purly to aid the
8427 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8428 * disable interrupt on NT.
8429 *
8430 * We need to check for force-flags that could've possible been altered since we last
8431 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8432 * see @bugref{6398}).
8433 *
8434 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8435 * to ring-3 before executing guest code.
8436 */
8437 pVmxTransient->fEFlags = ASMIntDisableFlags();
8438
8439 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8440 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8441 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8442 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8443 {
8444 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8445 {
8446 pVCpu->hm.s.Event.fPending = false;
8447
8448 /*
8449 * We've injected any pending events. This is really the point of no return (to ring-3).
8450 *
8451 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8452 * returns from this function, so don't enable them here.
8453 */
8454 return VINF_SUCCESS;
8455 }
8456
8457 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8458 rcStrict = VINF_EM_RAW_INTERRUPT;
8459 }
8460 else
8461 {
8462 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8463 rcStrict = VINF_EM_RAW_TO_R3;
8464 }
8465
8466 ASMSetFlags(pVmxTransient->fEFlags);
8467 VMMRZCallRing3Enable(pVCpu);
8468
8469 return rcStrict;
8470}
8471
8472
8473/**
8474 * Prepares to run guest code in VT-x and we've committed to doing so. This
8475 * means there is no backing out to ring-3 or anywhere else at this
8476 * point.
8477 *
8478 * @param pVCpu The cross context virtual CPU structure.
8479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8480 * out-of-sync. Make sure to update the required fields
8481 * before using them.
8482 * @param pVmxTransient Pointer to the VMX transient structure.
8483 *
8484 * @remarks Called with preemption disabled.
8485 * @remarks No-long-jump zone!!!
8486 */
8487static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8488{
8489 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8490 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8491 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8492
8493 /*
8494 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8495 */
8496 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8497 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8498
8499 PVM pVM = pVCpu->CTX_SUFF(pVM);
8500 if (!CPUMIsGuestFPUStateActive(pVCpu))
8501 {
8502 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8503 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8504 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8505 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8506 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8507 }
8508
8509 /*
8510 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8511 */
8512 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8513 && pVCpu->hm.s.vmx.cMsrs > 0)
8514 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8515
8516 /*
8517 * Re-save the host state bits as we may've been preempted (only happens when
8518 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8519 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8520 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8521 * See @bugref{8432}.
8522 */
8523 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8524 {
8525 int rc = hmR0VmxExportHostState(pVCpu);
8526 AssertRC(rc);
8527 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8528 }
8529 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8530
8531 /*
8532 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8533 */
8534 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8535 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8536 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8537
8538 /* Store status of the shared guest-host state at the time of VM-entry. */
8539#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8540 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8541 {
8542 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8543 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8544 }
8545 else
8546#endif
8547 {
8548 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8549 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8550 }
8551
8552 /*
8553 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8554 */
8555 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8556 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8557
8558 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8559 RTCPUID idCurrentCpu = pCpu->idCpu;
8560 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8561 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8562 {
8563 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8564 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8565 }
8566
8567 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8568 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8569 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8570 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8571
8572 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8573
8574 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8575 to start executing. */
8576
8577 /*
8578 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8579 */
8580 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8581 {
8582 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8583 {
8584 bool fMsrUpdated;
8585 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8586 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8587 &fMsrUpdated);
8588 AssertRC(rc2);
8589 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8590 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8591 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8592 }
8593 else
8594 {
8595 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8596 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8597 }
8598 }
8599
8600 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8601 {
8602 bool fMsrUpdated;
8603 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8604 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8605 &fMsrUpdated);
8606 AssertRC(rc2);
8607 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8608 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8609 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8610 }
8611
8612#ifdef VBOX_STRICT
8613 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8614 hmR0VmxCheckHostEferMsr(pVCpu);
8615 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8616#endif
8617#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8618 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8619 {
8620 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8621 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8622 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8623 }
8624#endif
8625}
8626
8627
8628/**
8629 * Performs some essential restoration of state after running guest code in
8630 * VT-x.
8631 *
8632 * @param pVCpu The cross context virtual CPU structure.
8633 * @param pVmxTransient Pointer to the VMX transient structure.
8634 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8635 *
8636 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8637 *
8638 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8639 * unconditionally when it is safe to do so.
8640 */
8641static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8642{
8643 uint64_t const uHostTsc = ASMReadTSC();
8644 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8645
8646 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8647 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8648 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8649 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8650 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8651 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8652
8653 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8654 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8655
8656 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8657 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8658 Assert(!ASMIntAreEnabled());
8659 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8660
8661#if HC_ARCH_BITS == 64
8662 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8663#endif
8664#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8665 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8666 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8667 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8668#else
8669 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8670#endif
8671#ifdef VBOX_STRICT
8672 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8673#endif
8674 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8675
8676 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8677 uint32_t uExitReason;
8678 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8679 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8680 AssertRC(rc);
8681 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8682 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8683
8684 if (rcVMRun == VINF_SUCCESS)
8685 {
8686 /*
8687 * Update the VM-exit history array here even if the VM-entry failed due to:
8688 * - Invalid guest state.
8689 * - MSR loading.
8690 * - Machine-check event.
8691 *
8692 * In any of the above cases we will still have a "valid" VM-exit reason
8693 * despite @a fVMEntryFailed being false.
8694 *
8695 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8696 *
8697 * Note! We don't have CS or RIP at this point. Will probably address that later
8698 * by amending the history entry added here.
8699 */
8700 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8701 UINT64_MAX, uHostTsc);
8702
8703 if (!pVmxTransient->fVMEntryFailed)
8704 {
8705 VMMRZCallRing3Enable(pVCpu);
8706
8707 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8708 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8709
8710#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8711 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8712 AssertRC(rc);
8713#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8714 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8715 AssertRC(rc);
8716#else
8717 /*
8718 * Import the guest-interruptibility state always as we need it while evaluating
8719 * injecting events on re-entry.
8720 *
8721 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8722 * checking for real-mode while exporting the state because all bits that cause
8723 * mode changes wrt CR0 are intercepted.
8724 */
8725 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8726 AssertRC(rc);
8727#endif
8728
8729 /*
8730 * Sync the TPR shadow with our APIC state.
8731 */
8732 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8733 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8734 {
8735 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8736 AssertRC(rc);
8737 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8738 }
8739
8740 return;
8741 }
8742 }
8743 else
8744 {
8745 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8746 }
8747
8748 VMMRZCallRing3Enable(pVCpu);
8749}
8750
8751
8752/**
8753 * Runs the guest code using VT-x the normal way.
8754 *
8755 * @returns VBox status code.
8756 * @param pVCpu The cross context virtual CPU structure.
8757 * @param pCtx Pointer to the guest-CPU context.
8758 *
8759 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8760 */
8761static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8762{
8763 VMXTRANSIENT VmxTransient;
8764 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8765 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8766 uint32_t cLoops = 0;
8767
8768 for (;; cLoops++)
8769 {
8770 Assert(!HMR0SuspendPending());
8771 HMVMX_ASSERT_CPU_SAFE();
8772
8773 /* Preparatory work for running guest code, this may force us to return
8774 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8775 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8776 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8777 if (rcStrict != VINF_SUCCESS)
8778 break;
8779
8780 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8781 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8782 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8783
8784 /* Restore any residual host-state and save any bits shared between host
8785 and guest into the guest-CPU state. Re-enables interrupts! */
8786 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8787
8788 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8789 if (RT_SUCCESS(rcRun))
8790 { /* very likely */ }
8791 else
8792 {
8793 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8794 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8795 return rcRun;
8796 }
8797
8798 /* Profile the VM-exit. */
8799 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8800 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8801 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8802 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8803 HMVMX_START_EXIT_DISPATCH_PROF();
8804
8805 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8806
8807 /* Handle the VM-exit. */
8808#ifdef HMVMX_USE_FUNCTION_TABLE
8809 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8810#else
8811 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8812#endif
8813 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8814 if (rcStrict == VINF_SUCCESS)
8815 {
8816 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8817 continue; /* likely */
8818 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8819 rcStrict = VINF_EM_RAW_INTERRUPT;
8820 }
8821 break;
8822 }
8823
8824 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8825 return rcStrict;
8826}
8827
8828
8829
8830/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8831 * probes.
8832 *
8833 * The following few functions and associated structure contains the bloat
8834 * necessary for providing detailed debug events and dtrace probes as well as
8835 * reliable host side single stepping. This works on the principle of
8836 * "subclassing" the normal execution loop and workers. We replace the loop
8837 * method completely and override selected helpers to add necessary adjustments
8838 * to their core operation.
8839 *
8840 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8841 * any performance for debug and analysis features.
8842 *
8843 * @{
8844 */
8845
8846/**
8847 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8848 * the debug run loop.
8849 */
8850typedef struct VMXRUNDBGSTATE
8851{
8852 /** The RIP we started executing at. This is for detecting that we stepped. */
8853 uint64_t uRipStart;
8854 /** The CS we started executing with. */
8855 uint16_t uCsStart;
8856
8857 /** Whether we've actually modified the 1st execution control field. */
8858 bool fModifiedProcCtls : 1;
8859 /** Whether we've actually modified the 2nd execution control field. */
8860 bool fModifiedProcCtls2 : 1;
8861 /** Whether we've actually modified the exception bitmap. */
8862 bool fModifiedXcptBitmap : 1;
8863
8864 /** We desire the modified the CR0 mask to be cleared. */
8865 bool fClearCr0Mask : 1;
8866 /** We desire the modified the CR4 mask to be cleared. */
8867 bool fClearCr4Mask : 1;
8868 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8869 uint32_t fCpe1Extra;
8870 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8871 uint32_t fCpe1Unwanted;
8872 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8873 uint32_t fCpe2Extra;
8874 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8875 uint32_t bmXcptExtra;
8876 /** The sequence number of the Dtrace provider settings the state was
8877 * configured against. */
8878 uint32_t uDtraceSettingsSeqNo;
8879 /** VM-exits to check (one bit per VM-exit). */
8880 uint32_t bmExitsToCheck[3];
8881
8882 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8883 uint32_t fProcCtlsInitial;
8884 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8885 uint32_t fProcCtls2Initial;
8886 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8887 uint32_t bmXcptInitial;
8888} VMXRUNDBGSTATE;
8889AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8890typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8891
8892
8893/**
8894 * Initializes the VMXRUNDBGSTATE structure.
8895 *
8896 * @param pVCpu The cross context virtual CPU structure of the
8897 * calling EMT.
8898 * @param pCtx The CPU register context to go with @a pVCpu.
8899 * @param pDbgState The structure to initialize.
8900 */
8901static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8902{
8903 pDbgState->uRipStart = pCtx->rip;
8904 pDbgState->uCsStart = pCtx->cs.Sel;
8905
8906 pDbgState->fModifiedProcCtls = false;
8907 pDbgState->fModifiedProcCtls2 = false;
8908 pDbgState->fModifiedXcptBitmap = false;
8909 pDbgState->fClearCr0Mask = false;
8910 pDbgState->fClearCr4Mask = false;
8911 pDbgState->fCpe1Extra = 0;
8912 pDbgState->fCpe1Unwanted = 0;
8913 pDbgState->fCpe2Extra = 0;
8914 pDbgState->bmXcptExtra = 0;
8915 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8916 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8917 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8918}
8919
8920
8921/**
8922 * Updates the VMSC fields with changes requested by @a pDbgState.
8923 *
8924 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8925 * immediately before executing guest code, i.e. when interrupts are disabled.
8926 * We don't check status codes here as we cannot easily assert or return in the
8927 * latter case.
8928 *
8929 * @param pVCpu The cross context virtual CPU structure.
8930 * @param pDbgState The debug state.
8931 */
8932static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8933{
8934 /*
8935 * Ensure desired flags in VMCS control fields are set.
8936 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8937 *
8938 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8939 * there should be no stale data in pCtx at this point.
8940 */
8941 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8942 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8943 {
8944 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8945 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8946 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8947 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8948 pDbgState->fModifiedProcCtls = true;
8949 }
8950
8951 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8952 {
8953 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8954 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8955 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8956 pDbgState->fModifiedProcCtls2 = true;
8957 }
8958
8959 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8960 {
8961 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8962 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8963 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8964 pDbgState->fModifiedXcptBitmap = true;
8965 }
8966
8967 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8968 {
8969 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8970 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8971 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8972 }
8973
8974 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8975 {
8976 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8977 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8978 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8979 }
8980}
8981
8982
8983static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8984{
8985 /*
8986 * Restore VM-exit control settings as we may not reenter this function the
8987 * next time around.
8988 */
8989 /* We reload the initial value, trigger what we can of recalculations the
8990 next time around. From the looks of things, that's all that's required atm. */
8991 if (pDbgState->fModifiedProcCtls)
8992 {
8993 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8994 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8995 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8996 AssertRCReturn(rc2, rc2);
8997 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8998 }
8999
9000 /* We're currently the only ones messing with this one, so just restore the
9001 cached value and reload the field. */
9002 if ( pDbgState->fModifiedProcCtls2
9003 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9004 {
9005 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9006 AssertRCReturn(rc2, rc2);
9007 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9008 }
9009
9010 /* If we've modified the exception bitmap, we restore it and trigger
9011 reloading and partial recalculation the next time around. */
9012 if (pDbgState->fModifiedXcptBitmap)
9013 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9014
9015 return rcStrict;
9016}
9017
9018
9019/**
9020 * Configures VM-exit controls for current DBGF and DTrace settings.
9021 *
9022 * This updates @a pDbgState and the VMCS execution control fields to reflect
9023 * the necessary VM-exits demanded by DBGF and DTrace.
9024 *
9025 * @param pVCpu The cross context virtual CPU structure.
9026 * @param pDbgState The debug state.
9027 * @param pVmxTransient Pointer to the VMX transient structure. May update
9028 * fUpdateTscOffsettingAndPreemptTimer.
9029 */
9030static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9031{
9032 /*
9033 * Take down the dtrace serial number so we can spot changes.
9034 */
9035 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9036 ASMCompilerBarrier();
9037
9038 /*
9039 * We'll rebuild most of the middle block of data members (holding the
9040 * current settings) as we go along here, so start by clearing it all.
9041 */
9042 pDbgState->bmXcptExtra = 0;
9043 pDbgState->fCpe1Extra = 0;
9044 pDbgState->fCpe1Unwanted = 0;
9045 pDbgState->fCpe2Extra = 0;
9046 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9047 pDbgState->bmExitsToCheck[i] = 0;
9048
9049 /*
9050 * Software interrupts (INT XXh) - no idea how to trigger these...
9051 */
9052 PVM pVM = pVCpu->CTX_SUFF(pVM);
9053 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9054 || VBOXVMM_INT_SOFTWARE_ENABLED())
9055 {
9056 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9057 }
9058
9059 /*
9060 * INT3 breakpoints - triggered by #BP exceptions.
9061 */
9062 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9063 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9064
9065 /*
9066 * Exception bitmap and XCPT events+probes.
9067 */
9068 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9069 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9070 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9071
9072 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9073 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9074 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9075 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9076 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9077 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9078 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9079 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9080 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9081 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9082 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9083 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9084 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9085 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9086 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9087 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9088 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9089 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9090
9091 if (pDbgState->bmXcptExtra)
9092 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9093
9094 /*
9095 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9096 *
9097 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9098 * So, when adding/changing/removing please don't forget to update it.
9099 *
9100 * Some of the macros are picking up local variables to save horizontal space,
9101 * (being able to see it in a table is the lesser evil here).
9102 */
9103#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9104 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9105 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9106#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9107 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9108 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9109 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9110 } else do { } while (0)
9111#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9112 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9113 { \
9114 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9115 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9116 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9117 } else do { } while (0)
9118#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9119 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9120 { \
9121 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9122 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9123 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9124 } else do { } while (0)
9125#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9126 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9127 { \
9128 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9129 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9130 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9131 } else do { } while (0)
9132
9133 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9134 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9135 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9136 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9137 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9138
9139 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9141 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9143 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9144 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9145 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9147 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9148 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9149 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9150 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9151 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9152 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9153 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9154 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9155 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9156 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9157 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9158 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9159 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9160 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9161 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9162 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9163 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9164 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9165 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9166 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9167 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9168 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9169 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9170 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9171 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9172 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9173 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9174 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9175
9176 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9177 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9178 {
9179 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9180 | CPUMCTX_EXTRN_CR4
9181 | CPUMCTX_EXTRN_APIC_TPR);
9182 AssertRC(rc);
9183
9184#if 0 /** @todo fix me */
9185 pDbgState->fClearCr0Mask = true;
9186 pDbgState->fClearCr4Mask = true;
9187#endif
9188 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9189 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9190 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9191 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9192 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9193 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9194 require clearing here and in the loop if we start using it. */
9195 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9196 }
9197 else
9198 {
9199 if (pDbgState->fClearCr0Mask)
9200 {
9201 pDbgState->fClearCr0Mask = false;
9202 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9203 }
9204 if (pDbgState->fClearCr4Mask)
9205 {
9206 pDbgState->fClearCr4Mask = false;
9207 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9208 }
9209 }
9210 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9211 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9212
9213 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9214 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9215 {
9216 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9217 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9218 }
9219 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9220 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9221
9222 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9223 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9224 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9225 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9226 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9227 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9228 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9229 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9230#if 0 /** @todo too slow, fix handler. */
9231 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9232#endif
9233 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9234
9235 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9236 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9237 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9238 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9239 {
9240 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9241 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9242 }
9243 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9244 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9245 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9246 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9247
9248 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9249 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9250 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9251 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9252 {
9253 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9254 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9255 }
9256 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9257 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9258 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9259 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9260
9261 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9262 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9263 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9264 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9265 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9266 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9267 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9268 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9269 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9270 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9271 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9272 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9273 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9274 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9275 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9276 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9277 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9278 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9279 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9280 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9281 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9282 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9283
9284#undef IS_EITHER_ENABLED
9285#undef SET_ONLY_XBM_IF_EITHER_EN
9286#undef SET_CPE1_XBM_IF_EITHER_EN
9287#undef SET_CPEU_XBM_IF_EITHER_EN
9288#undef SET_CPE2_XBM_IF_EITHER_EN
9289
9290 /*
9291 * Sanitize the control stuff.
9292 */
9293 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9294 if (pDbgState->fCpe2Extra)
9295 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9296 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9297 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9298 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9299 {
9300 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9301 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9302 }
9303
9304 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9305 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9306 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9307 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9308}
9309
9310
9311/**
9312 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9313 * appropriate.
9314 *
9315 * The caller has checked the VM-exit against the
9316 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9317 * already, so we don't have to do that either.
9318 *
9319 * @returns Strict VBox status code (i.e. informational status codes too).
9320 * @param pVCpu The cross context virtual CPU structure.
9321 * @param pMixedCtx Pointer to the guest-CPU context.
9322 * @param pVmxTransient Pointer to the VMX-transient structure.
9323 * @param uExitReason The VM-exit reason.
9324 *
9325 * @remarks The name of this function is displayed by dtrace, so keep it short
9326 * and to the point. No longer than 33 chars long, please.
9327 */
9328static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9329 uint32_t uExitReason)
9330{
9331 /*
9332 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9333 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9334 *
9335 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9336 * does. Must add/change/remove both places. Same ordering, please.
9337 *
9338 * Added/removed events must also be reflected in the next section
9339 * where we dispatch dtrace events.
9340 */
9341 bool fDtrace1 = false;
9342 bool fDtrace2 = false;
9343 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9344 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9345 uint32_t uEventArg = 0;
9346#define SET_EXIT(a_EventSubName) \
9347 do { \
9348 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9349 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9350 } while (0)
9351#define SET_BOTH(a_EventSubName) \
9352 do { \
9353 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9354 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9355 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9356 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9357 } while (0)
9358 switch (uExitReason)
9359 {
9360 case VMX_EXIT_MTF:
9361 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9362
9363 case VMX_EXIT_XCPT_OR_NMI:
9364 {
9365 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9366 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9367 {
9368 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9369 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9370 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9371 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9372 {
9373 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9374 {
9375 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9376 uEventArg = pVmxTransient->uExitIntErrorCode;
9377 }
9378 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9379 switch (enmEvent1)
9380 {
9381 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9382 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9383 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9384 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9385 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9386 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9387 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9388 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9389 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9390 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9391 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9392 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9393 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9394 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9395 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9396 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9397 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9398 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9399 default: break;
9400 }
9401 }
9402 else
9403 AssertFailed();
9404 break;
9405
9406 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9407 uEventArg = idxVector;
9408 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9409 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9410 break;
9411 }
9412 break;
9413 }
9414
9415 case VMX_EXIT_TRIPLE_FAULT:
9416 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9417 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9418 break;
9419 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9420 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9421 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9422 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9423 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9424
9425 /* Instruction specific VM-exits: */
9426 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9427 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9428 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9429 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9430 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9431 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9432 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9433 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9434 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9435 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9436 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9437 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9438 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9439 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9440 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9441 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9442 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9443 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9444 case VMX_EXIT_MOV_CRX:
9445 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9446 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9447 SET_BOTH(CRX_READ);
9448 else
9449 SET_BOTH(CRX_WRITE);
9450 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9451 break;
9452 case VMX_EXIT_MOV_DRX:
9453 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9454 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9455 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9456 SET_BOTH(DRX_READ);
9457 else
9458 SET_BOTH(DRX_WRITE);
9459 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9460 break;
9461 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9462 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9463 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9464 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9465 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9466 case VMX_EXIT_XDTR_ACCESS:
9467 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9468 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9469 {
9470 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9471 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9472 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9473 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9474 }
9475 break;
9476
9477 case VMX_EXIT_TR_ACCESS:
9478 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9479 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9480 {
9481 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9482 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9483 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9484 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9485 }
9486 break;
9487
9488 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9489 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9490 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9491 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9492 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9493 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9494 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9495 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9496 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9497 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9498 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9499
9500 /* Events that aren't relevant at this point. */
9501 case VMX_EXIT_EXT_INT:
9502 case VMX_EXIT_INT_WINDOW:
9503 case VMX_EXIT_NMI_WINDOW:
9504 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9505 case VMX_EXIT_PREEMPT_TIMER:
9506 case VMX_EXIT_IO_INSTR:
9507 break;
9508
9509 /* Errors and unexpected events. */
9510 case VMX_EXIT_INIT_SIGNAL:
9511 case VMX_EXIT_SIPI:
9512 case VMX_EXIT_IO_SMI:
9513 case VMX_EXIT_SMI:
9514 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9515 case VMX_EXIT_ERR_MSR_LOAD:
9516 case VMX_EXIT_ERR_MACHINE_CHECK:
9517 break;
9518
9519 default:
9520 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9521 break;
9522 }
9523#undef SET_BOTH
9524#undef SET_EXIT
9525
9526 /*
9527 * Dtrace tracepoints go first. We do them here at once so we don't
9528 * have to copy the guest state saving and stuff a few dozen times.
9529 * Down side is that we've got to repeat the switch, though this time
9530 * we use enmEvent since the probes are a subset of what DBGF does.
9531 */
9532 if (fDtrace1 || fDtrace2)
9533 {
9534 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9535 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9536 switch (enmEvent1)
9537 {
9538 /** @todo consider which extra parameters would be helpful for each probe. */
9539 case DBGFEVENT_END: break;
9540 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9542 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9549 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9550 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9551 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9552 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9553 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9558 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9559 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9560 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9568 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9569 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9570 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9571 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9572 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9573 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9574 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9599 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9606 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9607 }
9608 switch (enmEvent2)
9609 {
9610 /** @todo consider which extra parameters would be helpful for each probe. */
9611 case DBGFEVENT_END: break;
9612 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9614 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9615 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9617 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9622 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9623 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9624 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9625 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9626 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9627 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9628 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9631 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9632 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9633 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9634 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9635 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9636 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9637 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9638 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9639 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9652 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9653 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9664 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9665 }
9666 }
9667
9668 /*
9669 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9670 * the DBGF call will do a full check).
9671 *
9672 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9673 * Note! If we have to events, we prioritize the first, i.e. the instruction
9674 * one, in order to avoid event nesting.
9675 */
9676 PVM pVM = pVCpu->CTX_SUFF(pVM);
9677 if ( enmEvent1 != DBGFEVENT_END
9678 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9679 {
9680 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9681 if (rcStrict != VINF_SUCCESS)
9682 return rcStrict;
9683 }
9684 else if ( enmEvent2 != DBGFEVENT_END
9685 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9686 {
9687 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9688 if (rcStrict != VINF_SUCCESS)
9689 return rcStrict;
9690 }
9691
9692 return VINF_SUCCESS;
9693}
9694
9695
9696/**
9697 * Single-stepping VM-exit filtering.
9698 *
9699 * This is preprocessing the VM-exits and deciding whether we've gotten far
9700 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9701 * handling is performed.
9702 *
9703 * @returns Strict VBox status code (i.e. informational status codes too).
9704 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9705 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9706 * out-of-sync. Make sure to update the required
9707 * fields before using them.
9708 * @param pVmxTransient Pointer to the VMX-transient structure.
9709 * @param uExitReason The VM-exit reason.
9710 * @param pDbgState The debug state.
9711 */
9712DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9713 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9714{
9715 /*
9716 * Expensive (saves context) generic dtrace VM-exit probe.
9717 */
9718 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9719 { /* more likely */ }
9720 else
9721 {
9722 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9723 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9724 AssertRC(rc);
9725 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9726 }
9727
9728 /*
9729 * Check for host NMI, just to get that out of the way.
9730 */
9731 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9732 { /* normally likely */ }
9733 else
9734 {
9735 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9736 AssertRCReturn(rc2, rc2);
9737 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9738 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9739 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9740 }
9741
9742 /*
9743 * Check for single stepping event if we're stepping.
9744 */
9745 if (pVCpu->hm.s.fSingleInstruction)
9746 {
9747 switch (uExitReason)
9748 {
9749 case VMX_EXIT_MTF:
9750 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9751
9752 /* Various events: */
9753 case VMX_EXIT_XCPT_OR_NMI:
9754 case VMX_EXIT_EXT_INT:
9755 case VMX_EXIT_TRIPLE_FAULT:
9756 case VMX_EXIT_INT_WINDOW:
9757 case VMX_EXIT_NMI_WINDOW:
9758 case VMX_EXIT_TASK_SWITCH:
9759 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9760 case VMX_EXIT_APIC_ACCESS:
9761 case VMX_EXIT_EPT_VIOLATION:
9762 case VMX_EXIT_EPT_MISCONFIG:
9763 case VMX_EXIT_PREEMPT_TIMER:
9764
9765 /* Instruction specific VM-exits: */
9766 case VMX_EXIT_CPUID:
9767 case VMX_EXIT_GETSEC:
9768 case VMX_EXIT_HLT:
9769 case VMX_EXIT_INVD:
9770 case VMX_EXIT_INVLPG:
9771 case VMX_EXIT_RDPMC:
9772 case VMX_EXIT_RDTSC:
9773 case VMX_EXIT_RSM:
9774 case VMX_EXIT_VMCALL:
9775 case VMX_EXIT_VMCLEAR:
9776 case VMX_EXIT_VMLAUNCH:
9777 case VMX_EXIT_VMPTRLD:
9778 case VMX_EXIT_VMPTRST:
9779 case VMX_EXIT_VMREAD:
9780 case VMX_EXIT_VMRESUME:
9781 case VMX_EXIT_VMWRITE:
9782 case VMX_EXIT_VMXOFF:
9783 case VMX_EXIT_VMXON:
9784 case VMX_EXIT_MOV_CRX:
9785 case VMX_EXIT_MOV_DRX:
9786 case VMX_EXIT_IO_INSTR:
9787 case VMX_EXIT_RDMSR:
9788 case VMX_EXIT_WRMSR:
9789 case VMX_EXIT_MWAIT:
9790 case VMX_EXIT_MONITOR:
9791 case VMX_EXIT_PAUSE:
9792 case VMX_EXIT_XDTR_ACCESS:
9793 case VMX_EXIT_TR_ACCESS:
9794 case VMX_EXIT_INVEPT:
9795 case VMX_EXIT_RDTSCP:
9796 case VMX_EXIT_INVVPID:
9797 case VMX_EXIT_WBINVD:
9798 case VMX_EXIT_XSETBV:
9799 case VMX_EXIT_RDRAND:
9800 case VMX_EXIT_INVPCID:
9801 case VMX_EXIT_VMFUNC:
9802 case VMX_EXIT_RDSEED:
9803 case VMX_EXIT_XSAVES:
9804 case VMX_EXIT_XRSTORS:
9805 {
9806 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9807 | CPUMCTX_EXTRN_CS);
9808 AssertRCReturn(rc, rc);
9809 if ( pMixedCtx->rip != pDbgState->uRipStart
9810 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9811 return VINF_EM_DBG_STEPPED;
9812 break;
9813 }
9814
9815 /* Errors and unexpected events: */
9816 case VMX_EXIT_INIT_SIGNAL:
9817 case VMX_EXIT_SIPI:
9818 case VMX_EXIT_IO_SMI:
9819 case VMX_EXIT_SMI:
9820 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9821 case VMX_EXIT_ERR_MSR_LOAD:
9822 case VMX_EXIT_ERR_MACHINE_CHECK:
9823 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9824 break;
9825
9826 default:
9827 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9828 break;
9829 }
9830 }
9831
9832 /*
9833 * Check for debugger event breakpoints and dtrace probes.
9834 */
9835 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9836 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9837 {
9838 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9839 if (rcStrict != VINF_SUCCESS)
9840 return rcStrict;
9841 }
9842
9843 /*
9844 * Normal processing.
9845 */
9846#ifdef HMVMX_USE_FUNCTION_TABLE
9847 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9848#else
9849 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9850#endif
9851}
9852
9853
9854/**
9855 * Single steps guest code using VT-x.
9856 *
9857 * @returns Strict VBox status code (i.e. informational status codes too).
9858 * @param pVCpu The cross context virtual CPU structure.
9859 * @param pCtx Pointer to the guest-CPU context.
9860 *
9861 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9862 */
9863static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9864{
9865 VMXTRANSIENT VmxTransient;
9866 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9867
9868 /* Set HMCPU indicators. */
9869 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9870 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9871 pVCpu->hm.s.fDebugWantRdTscExit = false;
9872 pVCpu->hm.s.fUsingDebugLoop = true;
9873
9874 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9875 VMXRUNDBGSTATE DbgState;
9876 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9877 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9878
9879 /*
9880 * The loop.
9881 */
9882 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9883 for (uint32_t cLoops = 0; ; cLoops++)
9884 {
9885 Assert(!HMR0SuspendPending());
9886 HMVMX_ASSERT_CPU_SAFE();
9887 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9888
9889 /*
9890 * Preparatory work for running guest code, this may force us to return
9891 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9892 */
9893 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9894 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9895 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9896 if (rcStrict != VINF_SUCCESS)
9897 break;
9898
9899 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9900 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9901
9902 /*
9903 * Now we can run the guest code.
9904 */
9905 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9906
9907 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9908
9909 /*
9910 * Restore any residual host-state and save any bits shared between host
9911 * and guest into the guest-CPU state. Re-enables interrupts!
9912 */
9913 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9914
9915 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9916 if (RT_SUCCESS(rcRun))
9917 { /* very likely */ }
9918 else
9919 {
9920 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9921 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9922 return rcRun;
9923 }
9924
9925 /* Profile the VM-exit. */
9926 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9927 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9928 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9929 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9930 HMVMX_START_EXIT_DISPATCH_PROF();
9931
9932 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9933
9934 /*
9935 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9936 */
9937 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9938 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9939 if (rcStrict != VINF_SUCCESS)
9940 break;
9941 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9942 {
9943 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9944 rcStrict = VINF_EM_RAW_INTERRUPT;
9945 break;
9946 }
9947
9948 /*
9949 * Stepping: Did the RIP change, if so, consider it a single step.
9950 * Otherwise, make sure one of the TFs gets set.
9951 */
9952 if (fStepping)
9953 {
9954 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9955 | CPUMCTX_EXTRN_CS);
9956 AssertRC(rc);
9957 if ( pCtx->rip != DbgState.uRipStart
9958 || pCtx->cs.Sel != DbgState.uCsStart)
9959 {
9960 rcStrict = VINF_EM_DBG_STEPPED;
9961 break;
9962 }
9963 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9964 }
9965
9966 /*
9967 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9968 */
9969 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9970 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9971 }
9972
9973 /*
9974 * Clear the X86_EFL_TF if necessary.
9975 */
9976 if (pVCpu->hm.s.fClearTrapFlag)
9977 {
9978 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9979 AssertRC(rc);
9980 pVCpu->hm.s.fClearTrapFlag = false;
9981 pCtx->eflags.Bits.u1TF = 0;
9982 }
9983 /** @todo there seems to be issues with the resume flag when the monitor trap
9984 * flag is pending without being used. Seen early in bios init when
9985 * accessing APIC page in protected mode. */
9986
9987 /*
9988 * Restore VM-exit control settings as we may not reenter this function the
9989 * next time around.
9990 */
9991 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9992
9993 /* Restore HMCPU indicators. */
9994 pVCpu->hm.s.fUsingDebugLoop = false;
9995 pVCpu->hm.s.fDebugWantRdTscExit = false;
9996 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9997
9998 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9999 return rcStrict;
10000}
10001
10002
10003/** @} */
10004
10005
10006/**
10007 * Checks if any expensive dtrace probes are enabled and we should go to the
10008 * debug loop.
10009 *
10010 * @returns true if we should use debug loop, false if not.
10011 */
10012static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10013{
10014 /* It's probably faster to OR the raw 32-bit counter variables together.
10015 Since the variables are in an array and the probes are next to one
10016 another (more or less), we have good locality. So, better read
10017 eight-nine cache lines ever time and only have one conditional, than
10018 128+ conditionals, right? */
10019 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10020 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10021 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10022 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10023 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10024 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10025 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10026 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10027 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10028 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10029 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10030 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10031 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10032 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10033 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10034 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10035 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10036 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10037 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10038 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10039 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10040 ) != 0
10041 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10042 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10043 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10044 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10045 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10046 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10047 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10048 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10049 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10050 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10051 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10052 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10053 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10054 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10055 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10056 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10057 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10058 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10059 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10060 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10061 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10062 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10063 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10064 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10065 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10066 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10067 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10068 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10069 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10070 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10071 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10072 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10073 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10074 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10075 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10076 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10077 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10078 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10079 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10080 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10081 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10082 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10083 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10084 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10085 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10086 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10087 ) != 0
10088 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10089 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10090 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10091 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10092 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10093 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10094 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10095 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10096 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10097 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10098 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10099 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10100 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10101 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10102 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10103 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10104 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10105 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10106 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10107 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10108 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10109 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10110 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10111 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10112 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10113 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10114 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10115 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10116 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10117 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10118 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10119 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10120 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10121 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10122 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10123 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10124 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10125 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10126 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10127 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10128 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10129 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10130 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10131 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10132 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10133 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10134 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10135 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10136 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10137 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10138 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10139 ) != 0;
10140}
10141
10142
10143/**
10144 * Runs the guest code using VT-x.
10145 *
10146 * @returns Strict VBox status code (i.e. informational status codes too).
10147 * @param pVCpu The cross context virtual CPU structure.
10148 * @param pCtx Pointer to the guest-CPU context.
10149 */
10150VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10151{
10152 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10153 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10154 HMVMX_ASSERT_PREEMPT_SAFE();
10155
10156 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10157
10158 VBOXSTRICTRC rcStrict;
10159 if ( !pVCpu->hm.s.fUseDebugLoop
10160 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10161 && !DBGFIsStepping(pVCpu)
10162 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10163 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10164 else
10165 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10166
10167 if (rcStrict == VERR_EM_INTERPRETER)
10168 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10169 else if (rcStrict == VINF_EM_RESET)
10170 rcStrict = VINF_EM_TRIPLE_FAULT;
10171
10172 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10173 if (RT_FAILURE(rc2))
10174 {
10175 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10176 rcStrict = rc2;
10177 }
10178 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10179 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10180 return rcStrict;
10181}
10182
10183
10184#ifndef HMVMX_USE_FUNCTION_TABLE
10185DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10186{
10187#ifdef DEBUG_ramshankar
10188#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10189 do { \
10190 if (a_fSave != 0) \
10191 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10192 VBOXSTRICTRC rcStrict = a_CallExpr; \
10193 if (a_fSave != 0) \
10194 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10195 return rcStrict; \
10196 } while (0)
10197#else
10198# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10199#endif
10200 switch (rcReason)
10201 {
10202 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10203 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10204 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10205 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10206 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10207 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10208 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10209 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10210 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10211 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10212 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10213 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10214 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10215 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10216 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10217 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10218 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10219 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10220 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10221 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10222 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10223 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10224 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10225 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10226 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10227 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10228 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10229 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10236
10237 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10238 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10239 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10240 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10241 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10242 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10243 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10244 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10245 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10246
10247 case VMX_EXIT_VMCLEAR:
10248 case VMX_EXIT_VMLAUNCH:
10249 case VMX_EXIT_VMPTRLD:
10250 case VMX_EXIT_VMPTRST:
10251 case VMX_EXIT_VMREAD:
10252 case VMX_EXIT_VMRESUME:
10253 case VMX_EXIT_VMWRITE:
10254 case VMX_EXIT_VMXOFF:
10255 case VMX_EXIT_VMXON:
10256 case VMX_EXIT_INVEPT:
10257 case VMX_EXIT_INVVPID:
10258 case VMX_EXIT_VMFUNC:
10259 case VMX_EXIT_XSAVES:
10260 case VMX_EXIT_XRSTORS:
10261 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10262
10263 case VMX_EXIT_ENCLS:
10264 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10265 case VMX_EXIT_PML_FULL:
10266 default:
10267 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10268 }
10269#undef VMEXIT_CALL_RET
10270}
10271#endif /* !HMVMX_USE_FUNCTION_TABLE */
10272
10273
10274#ifdef VBOX_STRICT
10275/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10276# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10277 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10278
10279# define HMVMX_ASSERT_PREEMPT_CPUID() \
10280 do { \
10281 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10282 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10283 } while (0)
10284
10285# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10286 do { \
10287 AssertPtr(pVCpu); \
10288 AssertPtr(pMixedCtx); \
10289 AssertPtr(pVmxTransient); \
10290 Assert(pVmxTransient->fVMEntryFailed == false); \
10291 Assert(ASMIntAreEnabled()); \
10292 HMVMX_ASSERT_PREEMPT_SAFE(); \
10293 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10294 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10295 HMVMX_ASSERT_PREEMPT_SAFE(); \
10296 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10297 HMVMX_ASSERT_PREEMPT_CPUID(); \
10298 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10299 } while (0)
10300
10301# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10302 do { \
10303 Log4Func(("\n")); \
10304 } while (0)
10305#else /* nonstrict builds: */
10306# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10307 do { \
10308 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10309 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10310 } while (0)
10311# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10312#endif
10313
10314
10315/**
10316 * Advances the guest RIP by the specified number of bytes.
10317 *
10318 * @param pVCpu The cross context virtual CPU structure.
10319 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10320 * out-of-sync. Make sure to update the required fields
10321 * before using them.
10322 * @param cbInstr Number of bytes to advance the RIP by.
10323 *
10324 * @remarks No-long-jump zone!!!
10325 */
10326DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10327{
10328 /* Advance the RIP. */
10329 pMixedCtx->rip += cbInstr;
10330 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10331
10332 /* Update interrupt inhibition. */
10333 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10334 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10335 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10336}
10337
10338
10339/**
10340 * Advances the guest RIP after reading it from the VMCS.
10341 *
10342 * @returns VBox status code, no informational status codes.
10343 * @param pVCpu The cross context virtual CPU structure.
10344 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10345 * out-of-sync. Make sure to update the required fields
10346 * before using them.
10347 * @param pVmxTransient Pointer to the VMX transient structure.
10348 *
10349 * @remarks No-long-jump zone!!!
10350 */
10351static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10352{
10353 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10354 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10355 | CPUMCTX_EXTRN_RFLAGS);
10356 AssertRCReturn(rc, rc);
10357
10358 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10359
10360 /*
10361 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10362 * pending debug exception field as it takes care of priority of events.
10363 *
10364 * See Intel spec. 32.2.1 "Debug Exceptions".
10365 */
10366 if ( !pVCpu->hm.s.fSingleInstruction
10367 && pMixedCtx->eflags.Bits.u1TF)
10368 {
10369 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10370 AssertRCReturn(rc, rc);
10371 }
10372
10373 return VINF_SUCCESS;
10374}
10375
10376
10377/**
10378 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10379 * and update error record fields accordingly.
10380 *
10381 * @return VMX_IGS_* return codes.
10382 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10383 * wrong with the guest state.
10384 *
10385 * @param pVCpu The cross context virtual CPU structure.
10386 * @param pCtx Pointer to the guest-CPU state.
10387 *
10388 * @remarks This function assumes our cache of the VMCS controls
10389 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10390 */
10391static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10392{
10393#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10394#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10395 uError = (err); \
10396 break; \
10397 } else do { } while (0)
10398
10399 int rc;
10400 PVM pVM = pVCpu->CTX_SUFF(pVM);
10401 uint32_t uError = VMX_IGS_ERROR;
10402 uint32_t u32Val;
10403 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10404
10405 do
10406 {
10407 /*
10408 * CR0.
10409 */
10410 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10411 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10412 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10413 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10414 if (fUnrestrictedGuest)
10415 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10416
10417 uint32_t u32GuestCr0;
10418 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10419 AssertRCBreak(rc);
10420 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10421 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10422 if ( !fUnrestrictedGuest
10423 && (u32GuestCr0 & X86_CR0_PG)
10424 && !(u32GuestCr0 & X86_CR0_PE))
10425 {
10426 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10427 }
10428
10429 /*
10430 * CR4.
10431 */
10432 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10433 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10434
10435 uint32_t u32GuestCr4;
10436 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10437 AssertRCBreak(rc);
10438 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10439 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10440
10441 /*
10442 * IA32_DEBUGCTL MSR.
10443 */
10444 uint64_t u64Val;
10445 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10446 AssertRCBreak(rc);
10447 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10448 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10449 {
10450 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10451 }
10452 uint64_t u64DebugCtlMsr = u64Val;
10453
10454#ifdef VBOX_STRICT
10455 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10456 AssertRCBreak(rc);
10457 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10458#endif
10459 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10460
10461 /*
10462 * RIP and RFLAGS.
10463 */
10464 uint32_t u32Eflags;
10465#if HC_ARCH_BITS == 64
10466 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10467 AssertRCBreak(rc);
10468 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10469 if ( !fLongModeGuest
10470 || !pCtx->cs.Attr.n.u1Long)
10471 {
10472 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10473 }
10474 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10475 * must be identical if the "IA-32e mode guest" VM-entry
10476 * control is 1 and CS.L is 1. No check applies if the
10477 * CPU supports 64 linear-address bits. */
10478
10479 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10480 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10481 AssertRCBreak(rc);
10482 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10483 VMX_IGS_RFLAGS_RESERVED);
10484 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10485 u32Eflags = u64Val;
10486#else
10487 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10488 AssertRCBreak(rc);
10489 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10490 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10491#endif
10492
10493 if ( fLongModeGuest
10494 || ( fUnrestrictedGuest
10495 && !(u32GuestCr0 & X86_CR0_PE)))
10496 {
10497 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10498 }
10499
10500 uint32_t u32EntryInfo;
10501 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10502 AssertRCBreak(rc);
10503 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10504 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10505 {
10506 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10507 }
10508
10509 /*
10510 * 64-bit checks.
10511 */
10512#if HC_ARCH_BITS == 64
10513 if (fLongModeGuest)
10514 {
10515 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10516 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10517 }
10518
10519 if ( !fLongModeGuest
10520 && (u32GuestCr4 & X86_CR4_PCIDE))
10521 {
10522 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10523 }
10524
10525 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10526 * 51:32 beyond the processor's physical-address width are 0. */
10527
10528 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10529 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10530 {
10531 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10532 }
10533
10534 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10535 AssertRCBreak(rc);
10536 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10537
10538 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10539 AssertRCBreak(rc);
10540 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10541#endif
10542
10543 /*
10544 * PERF_GLOBAL MSR.
10545 */
10546 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10547 {
10548 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10549 AssertRCBreak(rc);
10550 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10551 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10552 }
10553
10554 /*
10555 * PAT MSR.
10556 */
10557 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10558 {
10559 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10560 AssertRCBreak(rc);
10561 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10562 for (unsigned i = 0; i < 8; i++)
10563 {
10564 uint8_t u8Val = (u64Val & 0xff);
10565 if ( u8Val != 0 /* UC */
10566 && u8Val != 1 /* WC */
10567 && u8Val != 4 /* WT */
10568 && u8Val != 5 /* WP */
10569 && u8Val != 6 /* WB */
10570 && u8Val != 7 /* UC- */)
10571 {
10572 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10573 }
10574 u64Val >>= 8;
10575 }
10576 }
10577
10578 /*
10579 * EFER MSR.
10580 */
10581 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10582 {
10583 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10584 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10585 AssertRCBreak(rc);
10586 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10587 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10588 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10589 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10590 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10591 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10592 || !(u32GuestCr0 & X86_CR0_PG)
10593 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10594 VMX_IGS_EFER_LMA_LME_MISMATCH);
10595 }
10596
10597 /*
10598 * Segment registers.
10599 */
10600 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10601 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10602 if (!(u32Eflags & X86_EFL_VM))
10603 {
10604 /* CS */
10605 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10606 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10607 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10608 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10609 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10610 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10611 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10612 /* CS cannot be loaded with NULL in protected mode. */
10613 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10614 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10615 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10616 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10617 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10618 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10619 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10620 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10621 else
10622 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10623
10624 /* SS */
10625 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10626 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10627 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10628 if ( !(pCtx->cr0 & X86_CR0_PE)
10629 || pCtx->cs.Attr.n.u4Type == 3)
10630 {
10631 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10632 }
10633 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10634 {
10635 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10636 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10637 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10638 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10639 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10640 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10641 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10642 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10643 }
10644
10645 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10646 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10647 {
10648 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10649 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10650 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10651 || pCtx->ds.Attr.n.u4Type > 11
10652 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10653 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10654 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10655 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10656 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10657 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10658 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10659 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10660 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10661 }
10662 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10663 {
10664 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10666 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10667 || pCtx->es.Attr.n.u4Type > 11
10668 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10669 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10670 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10671 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10672 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10673 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10674 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10675 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10676 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10677 }
10678 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10679 {
10680 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10681 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10682 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10683 || pCtx->fs.Attr.n.u4Type > 11
10684 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10685 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10686 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10687 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10688 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10689 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10690 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10691 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10692 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10693 }
10694 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10695 {
10696 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10697 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10698 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10699 || pCtx->gs.Attr.n.u4Type > 11
10700 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10701 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10702 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10703 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10704 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10705 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10706 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10707 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10708 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10709 }
10710 /* 64-bit capable CPUs. */
10711#if HC_ARCH_BITS == 64
10712 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10713 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10714 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10715 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10716 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10717 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10718 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10719 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10720 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10721 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10722 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10723#endif
10724 }
10725 else
10726 {
10727 /* V86 mode checks. */
10728 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10729 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10730 {
10731 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10732 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10733 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10734 }
10735 else
10736 {
10737 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10738 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10739 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10740 }
10741
10742 /* CS */
10743 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10744 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10745 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10746 /* SS */
10747 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10748 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10749 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10750 /* DS */
10751 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10752 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10753 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10754 /* ES */
10755 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10756 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10757 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10758 /* FS */
10759 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10760 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10761 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10762 /* GS */
10763 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10764 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10765 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10766 /* 64-bit capable CPUs. */
10767#if HC_ARCH_BITS == 64
10768 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10769 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10770 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10771 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10772 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10773 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10774 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10775 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10776 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10777 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10778 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10779#endif
10780 }
10781
10782 /*
10783 * TR.
10784 */
10785 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10786 /* 64-bit capable CPUs. */
10787#if HC_ARCH_BITS == 64
10788 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10789#endif
10790 if (fLongModeGuest)
10791 {
10792 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10793 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10794 }
10795 else
10796 {
10797 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10798 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10799 VMX_IGS_TR_ATTR_TYPE_INVALID);
10800 }
10801 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10802 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10803 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10804 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10805 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10806 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10807 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10808 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10809
10810 /*
10811 * GDTR and IDTR.
10812 */
10813#if HC_ARCH_BITS == 64
10814 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10815 AssertRCBreak(rc);
10816 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10817
10818 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10819 AssertRCBreak(rc);
10820 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10821#endif
10822
10823 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10824 AssertRCBreak(rc);
10825 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10826
10827 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10828 AssertRCBreak(rc);
10829 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10830
10831 /*
10832 * Guest Non-Register State.
10833 */
10834 /* Activity State. */
10835 uint32_t u32ActivityState;
10836 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10837 AssertRCBreak(rc);
10838 HMVMX_CHECK_BREAK( !u32ActivityState
10839 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10840 VMX_IGS_ACTIVITY_STATE_INVALID);
10841 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10842 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10843 uint32_t u32IntrState;
10844 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10845 AssertRCBreak(rc);
10846 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10847 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10848 {
10849 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10850 }
10851
10852 /** @todo Activity state and injecting interrupts. Left as a todo since we
10853 * currently don't use activity states but ACTIVE. */
10854
10855 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10856 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10857
10858 /* Guest interruptibility-state. */
10859 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10860 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10861 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10862 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10863 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10864 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10865 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10866 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10867 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10868 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10869 {
10870 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10871 {
10872 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10873 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10874 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10875 }
10876 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10877 {
10878 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10879 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10880 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10881 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10882 }
10883 }
10884 /** @todo Assumes the processor is not in SMM. */
10885 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10886 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10887 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10888 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10889 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10890 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10891 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10892 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10893 {
10894 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10895 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10896 }
10897
10898 /* Pending debug exceptions. */
10899#if HC_ARCH_BITS == 64
10900 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10901 AssertRCBreak(rc);
10902 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10903 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10904 u32Val = u64Val; /* For pending debug exceptions checks below. */
10905#else
10906 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10907 AssertRCBreak(rc);
10908 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10909 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10910#endif
10911
10912 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10913 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10914 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10915 {
10916 if ( (u32Eflags & X86_EFL_TF)
10917 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10918 {
10919 /* Bit 14 is PendingDebug.BS. */
10920 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10921 }
10922 if ( !(u32Eflags & X86_EFL_TF)
10923 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10924 {
10925 /* Bit 14 is PendingDebug.BS. */
10926 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10927 }
10928 }
10929
10930 /* VMCS link pointer. */
10931 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10932 AssertRCBreak(rc);
10933 if (u64Val != UINT64_C(0xffffffffffffffff))
10934 {
10935 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10936 /** @todo Bits beyond the processor's physical-address width MBZ. */
10937 /** @todo 32-bit located in memory referenced by value of this field (as a
10938 * physical address) must contain the processor's VMCS revision ID. */
10939 /** @todo SMM checks. */
10940 }
10941
10942 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10943 * not using Nested Paging? */
10944 if ( pVM->hm.s.fNestedPaging
10945 && !fLongModeGuest
10946 && CPUMIsGuestInPAEModeEx(pCtx))
10947 {
10948 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10949 AssertRCBreak(rc);
10950 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10951
10952 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10953 AssertRCBreak(rc);
10954 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10955
10956 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10957 AssertRCBreak(rc);
10958 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10959
10960 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10961 AssertRCBreak(rc);
10962 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10963 }
10964
10965 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10966 if (uError == VMX_IGS_ERROR)
10967 uError = VMX_IGS_REASON_NOT_FOUND;
10968 } while (0);
10969
10970 pVCpu->hm.s.u32HMError = uError;
10971 return uError;
10972
10973#undef HMVMX_ERROR_BREAK
10974#undef HMVMX_CHECK_BREAK
10975}
10976
10977/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10978/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10979/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10980
10981/** @name VM-exit handlers.
10982 * @{
10983 */
10984
10985/**
10986 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10987 */
10988HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10989{
10990 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10992 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10993 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10994 return VINF_SUCCESS;
10995 return VINF_EM_RAW_INTERRUPT;
10996}
10997
10998
10999/**
11000 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11001 */
11002HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11003{
11004 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11005 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11006
11007 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11008 AssertRCReturn(rc, rc);
11009
11010 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11011 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11012 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11013 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11014
11015 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11016 {
11017 /*
11018 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11019 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11020 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11021 *
11022 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11023 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11024 */
11025 VMXDispatchHostNmi();
11026 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11027 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11028 return VINF_SUCCESS;
11029 }
11030
11031 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11032 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11033 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11034 { /* likely */ }
11035 else
11036 {
11037 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11038 rcStrictRc1 = VINF_SUCCESS;
11039 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11040 return rcStrictRc1;
11041 }
11042
11043 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11044 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11045 switch (uIntType)
11046 {
11047 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11048 Assert(uVector == X86_XCPT_DB);
11049 RT_FALL_THRU();
11050 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11051 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11052 RT_FALL_THRU();
11053 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11054 {
11055 /*
11056 * If there's any exception caused as a result of event injection, the resulting
11057 * secondary/final execption will be pending, we shall continue guest execution
11058 * after injecting the event. The page-fault case is complicated and we manually
11059 * handle any currently pending event in hmR0VmxExitXcptPF.
11060 */
11061 if (!pVCpu->hm.s.Event.fPending)
11062 { /* likely */ }
11063 else if (uVector != X86_XCPT_PF)
11064 {
11065 rc = VINF_SUCCESS;
11066 break;
11067 }
11068
11069 switch (uVector)
11070 {
11071 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11072 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11073 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11074 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11075 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11076 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11077
11078 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11079 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11080 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11081 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11082 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11083 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11084 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11085 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11086 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11087 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11088 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11089 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11090 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11091 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11092 default:
11093 {
11094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11095 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11096 {
11097 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11098 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11099 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11100
11101 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11102 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11103 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11104 AssertRCReturn(rc, rc);
11105 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11106 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11107 0 /* GCPtrFaultAddress */);
11108 }
11109 else
11110 {
11111 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11112 pVCpu->hm.s.u32HMError = uVector;
11113 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11114 }
11115 break;
11116 }
11117 }
11118 break;
11119 }
11120
11121 default:
11122 {
11123 pVCpu->hm.s.u32HMError = uExitIntInfo;
11124 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11125 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11126 break;
11127 }
11128 }
11129 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11130 return rc;
11131}
11132
11133
11134/**
11135 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11136 */
11137HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11138{
11139 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11140
11141 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11142 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11143
11144 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11146 return VINF_SUCCESS;
11147}
11148
11149
11150/**
11151 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11152 */
11153HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11154{
11155 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11156 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11157 {
11158 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11159 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11160 }
11161
11162 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11163
11164 /*
11165 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11166 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11167 */
11168 uint32_t fIntrState = 0;
11169 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11170 AssertRCReturn(rc, rc);
11171
11172 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11173 if ( fBlockSti
11174 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11175 {
11176 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11177 }
11178
11179 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11180 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11181
11182 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11183 return VINF_SUCCESS;
11184}
11185
11186
11187/**
11188 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11189 */
11190HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11191{
11192 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11193 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11194}
11195
11196
11197/**
11198 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11199 */
11200HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11201{
11202 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11203 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11204}
11205
11206
11207/**
11208 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11209 */
11210HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11211{
11212 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11213 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11214
11215 /*
11216 * Get the state we need and update the exit history entry.
11217 */
11218 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11219 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11220 | CPUMCTX_EXTRN_CS);
11221 AssertRCReturn(rc, rc);
11222
11223 VBOXSTRICTRC rcStrict;
11224 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11225 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11226 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11227 if (!pExitRec)
11228 {
11229 /*
11230 * Regular CPUID instruction execution.
11231 */
11232 PVM pVM = pVCpu->CTX_SUFF(pVM);
11233 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11234 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11235 {
11236 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11237 Assert(pVmxTransient->cbInstr == 2);
11238 }
11239 else
11240 {
11241 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11242 rcStrict = VERR_EM_INTERPRETER;
11243 }
11244 }
11245 else
11246 {
11247 /*
11248 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11249 */
11250 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11251 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11252 AssertRCReturn(rc2, rc2);
11253
11254 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11255 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11256
11257 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11258 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11259
11260 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11261 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11262 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11263 }
11264 return VBOXSTRICTRC_TODO(rcStrict);
11265}
11266
11267
11268/**
11269 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11270 */
11271HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11272{
11273 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11274 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11275 AssertRCReturn(rc, rc);
11276
11277 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11278 return VINF_EM_RAW_EMULATE_INSTR;
11279
11280 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11281 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11282}
11283
11284
11285/**
11286 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11287 */
11288HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11289{
11290 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11291 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11292 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11293 AssertRCReturn(rc, rc);
11294
11295 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11296 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11297 {
11298 /* If we get a spurious VM-exit when offsetting is enabled,
11299 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11300 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11301 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11302 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11303 | HM_CHANGED_GUEST_RFLAGS);
11304 }
11305 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11306 {
11307 rcStrict = VINF_SUCCESS;
11308 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11309 }
11310 return rcStrict;
11311}
11312
11313
11314/**
11315 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11316 */
11317HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11318{
11319 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11320 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11321 | CPUMCTX_EXTRN_TSC_AUX);
11322 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11323 AssertRCReturn(rc, rc);
11324
11325 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11326 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11327 {
11328 /* If we get a spurious VM-exit when offsetting is enabled,
11329 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11330 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11331 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11332 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11333 | HM_CHANGED_GUEST_RFLAGS);
11334 }
11335 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11336 {
11337 rcStrict = VINF_SUCCESS;
11338 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11339 }
11340 return rcStrict;
11341}
11342
11343
11344/**
11345 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11346 */
11347HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11348{
11349 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11350 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11351 | CPUMCTX_EXTRN_CR0
11352 | CPUMCTX_EXTRN_RFLAGS
11353 | CPUMCTX_EXTRN_SS);
11354 AssertRCReturn(rc, rc);
11355
11356 PVM pVM = pVCpu->CTX_SUFF(pVM);
11357 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11358 if (RT_LIKELY(rc == VINF_SUCCESS))
11359 {
11360 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11361 Assert(pVmxTransient->cbInstr == 2);
11362 }
11363 else
11364 {
11365 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11366 rc = VERR_EM_INTERPRETER;
11367 }
11368 return rc;
11369}
11370
11371
11372/**
11373 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11374 */
11375HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11376{
11377 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11378
11379 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11380 if (EMAreHypercallInstructionsEnabled(pVCpu))
11381 {
11382 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11383 | CPUMCTX_EXTRN_RFLAGS
11384 | CPUMCTX_EXTRN_CR0
11385 | CPUMCTX_EXTRN_SS
11386 | CPUMCTX_EXTRN_CS
11387 | CPUMCTX_EXTRN_EFER);
11388 AssertRCReturn(rc, rc);
11389
11390 /* Perform the hypercall. */
11391 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11392 if (rcStrict == VINF_SUCCESS)
11393 {
11394 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11395 AssertRCReturn(rc, rc);
11396 }
11397 else
11398 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11399 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11400 || RT_FAILURE(rcStrict));
11401
11402 /* If the hypercall changes anything other than guest's general-purpose registers,
11403 we would need to reload the guest changed bits here before VM-entry. */
11404 }
11405 else
11406 Log4Func(("Hypercalls not enabled\n"));
11407
11408 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11409 if (RT_FAILURE(rcStrict))
11410 {
11411 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11412 rcStrict = VINF_SUCCESS;
11413 }
11414
11415 return rcStrict;
11416}
11417
11418
11419/**
11420 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11421 */
11422HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11423{
11424 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11425 PVM pVM = pVCpu->CTX_SUFF(pVM);
11426 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11427
11428 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11429 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11430 AssertRCReturn(rc, rc);
11431
11432 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11433 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11434 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11435 else
11436 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11437 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11438 return rcStrict;
11439}
11440
11441
11442/**
11443 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11444 */
11445HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11446{
11447 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11448 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11449 | CPUMCTX_EXTRN_RFLAGS
11450 | CPUMCTX_EXTRN_SS);
11451 AssertRCReturn(rc, rc);
11452
11453 PVM pVM = pVCpu->CTX_SUFF(pVM);
11454 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11455 if (RT_LIKELY(rc == VINF_SUCCESS))
11456 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11457 else
11458 {
11459 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11460 rc = VERR_EM_INTERPRETER;
11461 }
11462 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11463 return rc;
11464}
11465
11466
11467/**
11468 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11469 */
11470HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11471{
11472 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11473 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11474 | CPUMCTX_EXTRN_RFLAGS
11475 | CPUMCTX_EXTRN_SS);
11476 AssertRCReturn(rc, rc);
11477
11478 PVM pVM = pVCpu->CTX_SUFF(pVM);
11479 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11480 rc = VBOXSTRICTRC_VAL(rc2);
11481 if (RT_LIKELY( rc == VINF_SUCCESS
11482 || rc == VINF_EM_HALT))
11483 {
11484 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11485 AssertRCReturn(rc3, rc3);
11486
11487 if ( rc == VINF_EM_HALT
11488 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11489 rc = VINF_SUCCESS;
11490 }
11491 else
11492 {
11493 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11494 rc = VERR_EM_INTERPRETER;
11495 }
11496 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11497 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11498 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11499 return rc;
11500}
11501
11502
11503/**
11504 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11505 */
11506HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11507{
11508 /*
11509 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11510 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11511 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11512 * VMX root operation. If we get here, something funny is going on.
11513 *
11514 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11515 */
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11518 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11519}
11520
11521
11522/**
11523 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11524 */
11525HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11526{
11527 /*
11528 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11529 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11530 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11531 * an SMI. If we get here, something funny is going on.
11532 *
11533 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11534 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11535 */
11536 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11537 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11538 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11539}
11540
11541
11542/**
11543 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11544 */
11545HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11546{
11547 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11548 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11549 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11550 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11551}
11552
11553
11554/**
11555 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11556 */
11557HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11558{
11559 /*
11560 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11561 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11562 * See Intel spec. 25.3 "Other Causes of VM-exits".
11563 */
11564 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11565 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11566 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11567}
11568
11569
11570/**
11571 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11572 * VM-exit.
11573 */
11574HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11575{
11576 /*
11577 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11578 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11579 *
11580 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11581 * See Intel spec. "23.8 Restrictions on VMX operation".
11582 */
11583 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11584 return VINF_SUCCESS;
11585}
11586
11587
11588/**
11589 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11590 * VM-exit.
11591 */
11592HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11593{
11594 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11595 return VINF_EM_RESET;
11596}
11597
11598
11599/**
11600 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11601 */
11602HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11603{
11604 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11605 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11606
11607 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11608 AssertRCReturn(rc, rc);
11609
11610 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11611 rc = VINF_SUCCESS;
11612 else
11613 rc = VINF_EM_HALT;
11614
11615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11616 if (rc != VINF_SUCCESS)
11617 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11618 return rc;
11619}
11620
11621
11622/**
11623 * VM-exit handler for instructions that result in a \#UD exception delivered to
11624 * the guest.
11625 */
11626HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11627{
11628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11629 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11630 return VINF_SUCCESS;
11631}
11632
11633
11634/**
11635 * VM-exit handler for expiry of the VMX preemption timer.
11636 */
11637HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11638{
11639 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11640
11641 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11642 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11643
11644 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11645 PVM pVM = pVCpu->CTX_SUFF(pVM);
11646 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11647 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11648 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11649}
11650
11651
11652/**
11653 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11654 */
11655HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11656{
11657 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11658
11659 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11660 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11661 | CPUMCTX_EXTRN_CR4);
11662 AssertRCReturn(rc, rc);
11663
11664 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11665 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11666 : HM_CHANGED_XCPT_RAISED_MASK);
11667
11668 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11669
11670 return rcStrict;
11671}
11672
11673
11674/**
11675 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11676 */
11677HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11678{
11679 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11680 /** @todo Use VM-exit instruction information. */
11681 return VERR_EM_INTERPRETER;
11682}
11683
11684
11685/**
11686 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11687 * Error VM-exit.
11688 */
11689HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11690{
11691 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11692 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11693 AssertRCReturn(rc, rc);
11694
11695 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11696 NOREF(uInvalidReason);
11697
11698#ifdef VBOX_STRICT
11699 uint32_t fIntrState;
11700 RTHCUINTREG uHCReg;
11701 uint64_t u64Val;
11702 uint32_t u32Val;
11703
11704 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11705 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11706 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11707 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11708 AssertRCReturn(rc, rc);
11709
11710 Log4(("uInvalidReason %u\n", uInvalidReason));
11711 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11712 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11713 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11714 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11715
11716 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11717 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11718 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11719 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11720 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11721 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11722 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11723 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11724 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11725 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11726 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11727 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11728
11729 hmR0DumpRegs(pVCpu, pMixedCtx);
11730#else
11731 NOREF(pVmxTransient);
11732#endif
11733
11734 return VERR_VMX_INVALID_GUEST_STATE;
11735}
11736
11737
11738/**
11739 * VM-exit handler for VM-entry failure due to an MSR-load
11740 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11741 */
11742HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11743{
11744 NOREF(pVmxTransient);
11745 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11746 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11747}
11748
11749
11750/**
11751 * VM-exit handler for VM-entry failure due to a machine-check event
11752 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11753 */
11754HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11755{
11756 NOREF(pVmxTransient);
11757 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11758 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11759}
11760
11761
11762/**
11763 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11764 * theory.
11765 */
11766HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11767{
11768 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11769 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11770 return VERR_VMX_UNDEFINED_EXIT_CODE;
11771}
11772
11773
11774/**
11775 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11776 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11777 * Conditional VM-exit.
11778 */
11779HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11780{
11781 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11782
11783 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11784 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11785 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11786 return VERR_EM_INTERPRETER;
11787 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11788 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11789}
11790
11791
11792/**
11793 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11794 */
11795HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11796{
11797 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11798
11799 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11800 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11801 return VERR_EM_INTERPRETER;
11802 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11803 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11804}
11805
11806
11807/**
11808 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11809 */
11810HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11811{
11812 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11813
11814 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11815 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11816 | CPUMCTX_EXTRN_RFLAGS
11817 | CPUMCTX_EXTRN_SS
11818 | CPUMCTX_EXTRN_FS
11819 | CPUMCTX_EXTRN_GS);
11820 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11821 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11822 AssertRCReturn(rc, rc);
11823 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11824
11825#ifdef VBOX_STRICT
11826 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11827 {
11828 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11829 && pMixedCtx->ecx != MSR_K6_EFER)
11830 {
11831 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11832 pMixedCtx->ecx));
11833 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11834 }
11835 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11836 {
11837 VMXMSREXITREAD enmRead;
11838 VMXMSREXITWRITE enmWrite;
11839 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11840 AssertRCReturn(rc2, rc2);
11841 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11842 {
11843 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11844 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11845 }
11846 }
11847 }
11848#endif
11849
11850 PVM pVM = pVCpu->CTX_SUFF(pVM);
11851 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11852 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11853 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11854 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11855 if (RT_SUCCESS(rc))
11856 {
11857 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11858 Assert(pVmxTransient->cbInstr == 2);
11859 }
11860 return rc;
11861}
11862
11863
11864/**
11865 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11866 */
11867HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11868{
11869 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11870 PVM pVM = pVCpu->CTX_SUFF(pVM);
11871 int rc = VINF_SUCCESS;
11872
11873 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. FS, GS (base) can be accessed by MSR writes. */
11874 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11875 | CPUMCTX_EXTRN_RFLAGS
11876 | CPUMCTX_EXTRN_SS
11877 | CPUMCTX_EXTRN_FS
11878 | CPUMCTX_EXTRN_GS);
11879 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11880 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11881 AssertRCReturn(rc, rc);
11882 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11883
11884 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11885 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11886 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11887
11888 if (RT_SUCCESS(rc))
11889 {
11890 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11891
11892 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11893 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11894 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11895 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11896 {
11897 /*
11898 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11899 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11900 * EMInterpretWrmsr() changes it.
11901 */
11902 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11903 }
11904 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11905 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11906 else if (pMixedCtx->ecx == MSR_K6_EFER)
11907 {
11908 /*
11909 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11910 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11911 * the other bits as well, SCE and NXE. See @bugref{7368}.
11912 */
11913 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11914 | HM_CHANGED_VMX_ENTRY_CTLS
11915 | HM_CHANGED_VMX_EXIT_CTLS);
11916 }
11917
11918 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11919 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11920 {
11921 switch (pMixedCtx->ecx)
11922 {
11923 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11924 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11925 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11926 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11927 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11928 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11929 default:
11930 {
11931 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11932 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11933 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11934 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11935 break;
11936 }
11937 }
11938 }
11939#ifdef VBOX_STRICT
11940 else
11941 {
11942 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11943 switch (pMixedCtx->ecx)
11944 {
11945 case MSR_IA32_SYSENTER_CS:
11946 case MSR_IA32_SYSENTER_EIP:
11947 case MSR_IA32_SYSENTER_ESP:
11948 case MSR_K8_FS_BASE:
11949 case MSR_K8_GS_BASE:
11950 {
11951 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11952 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11953 }
11954
11955 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11956 default:
11957 {
11958 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11959 {
11960 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11961 if (pMixedCtx->ecx != MSR_K6_EFER)
11962 {
11963 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11964 pMixedCtx->ecx));
11965 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11966 }
11967 }
11968
11969 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11970 {
11971 VMXMSREXITREAD enmRead;
11972 VMXMSREXITWRITE enmWrite;
11973 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11974 AssertRCReturn(rc2, rc2);
11975 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11976 {
11977 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11978 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
11979 }
11980 }
11981 break;
11982 }
11983 }
11984 }
11985#endif /* VBOX_STRICT */
11986 }
11987 return rc;
11988}
11989
11990
11991/**
11992 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11993 */
11994HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11995{
11996 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11997 /** @todo The guest has likely hit a contended spinlock. We might want to
11998 * poke a schedule different guest VCPU. */
11999 return VINF_EM_RAW_INTERRUPT;
12000}
12001
12002
12003/**
12004 * VM-exit handler for when the TPR value is lowered below the specified
12005 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12006 */
12007HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12008{
12009 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12010 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12011
12012 /*
12013 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12014 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12015 */
12016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12017 return VINF_SUCCESS;
12018}
12019
12020
12021/**
12022 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12023 * VM-exit.
12024 *
12025 * @retval VINF_SUCCESS when guest execution can continue.
12026 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12027 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12028 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12029 * interpreter.
12030 */
12031HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12032{
12033 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12034 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12035
12036 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12037 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12038 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12039 AssertRCReturn(rc, rc);
12040
12041 VBOXSTRICTRC rcStrict;
12042 PVM pVM = pVCpu->CTX_SUFF(pVM);
12043 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12044 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12045 switch (uAccessType)
12046 {
12047 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12048 {
12049 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12050 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12051 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12052 AssertMsg( rcStrict == VINF_SUCCESS
12053 || rcStrict == VINF_IEM_RAISED_XCPT
12054 || rcStrict == VINF_PGM_CHANGE_MODE
12055 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12056
12057 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12058 {
12059 case 0:
12060 {
12061 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12062 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12063 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12064 break;
12065 }
12066
12067 case 2:
12068 {
12069 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12070 /* Nothing to do here, CR2 it's not part of the VMCS. */
12071 break;
12072 }
12073
12074 case 3:
12075 {
12076 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12077 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12078 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12079 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12080 break;
12081 }
12082
12083 case 4:
12084 {
12085 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12086 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12087 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12088 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12089 break;
12090 }
12091
12092 case 8:
12093 {
12094 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12095 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12096 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12097 break;
12098 }
12099 default:
12100 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12101 break;
12102 }
12103 break;
12104 }
12105
12106 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12107 {
12108 Assert( !pVM->hm.s.fNestedPaging
12109 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12110 || pVCpu->hm.s.fUsingDebugLoop
12111 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12112 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12113 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12114 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12115
12116 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12117 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12118 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12119 AssertMsg( rcStrict == VINF_SUCCESS
12120 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12121#ifdef VBOX_WITH_STATISTICS
12122 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12123 {
12124 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12125 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12126 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12127 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12128 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12129 }
12130#endif
12131 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12132 VBOXSTRICTRC_VAL(rcStrict)));
12133 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12134 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12135 break;
12136 }
12137
12138 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12139 {
12140 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12141 AssertMsg( rcStrict == VINF_SUCCESS
12142 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12143
12144 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12145 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12146 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12147 break;
12148 }
12149
12150 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12151 {
12152 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12153 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12154 AssertMsg( rcStrict == VINF_SUCCESS
12155 || rcStrict == VINF_IEM_RAISED_XCPT
12156 || rcStrict == VINF_PGM_CHANGE_MODE,
12157 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12158
12159 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12160 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12161 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12162 break;
12163 }
12164
12165 default:
12166 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12167 VERR_VMX_UNEXPECTED_EXCEPTION);
12168 }
12169
12170 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12171 : HM_CHANGED_XCPT_RAISED_MASK);
12172 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12173 NOREF(pVM);
12174 return rcStrict;
12175}
12176
12177
12178/**
12179 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12180 * VM-exit.
12181 */
12182HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12183{
12184 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12185 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12186 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12187
12188 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12189 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12190 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12191 | CPUMCTX_EXTRN_SREG_MASK
12192 | CPUMCTX_EXTRN_EFER);
12193 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12194 AssertRCReturn(rc, rc);
12195
12196 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12197 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12198 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12199 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12200 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12201 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12202 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12203 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12204 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12205
12206 /*
12207 * Update exit history to see if this exit can be optimized.
12208 */
12209 VBOXSTRICTRC rcStrict;
12210 PCEMEXITREC pExitRec = NULL;
12211 if ( !fGstStepping
12212 && !fDbgStepping)
12213 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12214 !fIOString
12215 ? !fIOWrite
12216 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12217 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12218 : !fIOWrite
12219 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12220 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12221 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12222 if (!pExitRec)
12223 {
12224 /* I/O operation lookup arrays. */
12225 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12226 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12227 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12228 uint32_t const cbInstr = pVmxTransient->cbInstr;
12229 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12230 PVM pVM = pVCpu->CTX_SUFF(pVM);
12231 if (fIOString)
12232 {
12233 /*
12234 * INS/OUTS - I/O String instruction.
12235 *
12236 * Use instruction-information if available, otherwise fall back on
12237 * interpreting the instruction.
12238 */
12239 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12240 fIOWrite ? 'w' : 'r'));
12241 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12242 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12243 {
12244 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12245 AssertRCReturn(rc2, rc2);
12246 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12247 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12248 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12249 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12250 if (fIOWrite)
12251 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12252 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12253 else
12254 {
12255 /*
12256 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12257 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12258 * See Intel Instruction spec. for "INS".
12259 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12260 */
12261 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12262 }
12263 }
12264 else
12265 rcStrict = IEMExecOne(pVCpu);
12266
12267 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12268 fUpdateRipAlready = true;
12269 }
12270 else
12271 {
12272 /*
12273 * IN/OUT - I/O instruction.
12274 */
12275 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12276 fIOWrite ? 'w' : 'r'));
12277 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12278 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12279 if (fIOWrite)
12280 {
12281 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12282 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12283 }
12284 else
12285 {
12286 uint32_t u32Result = 0;
12287 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12288 if (IOM_SUCCESS(rcStrict))
12289 {
12290 /* Save result of I/O IN instr. in AL/AX/EAX. */
12291 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12292 }
12293 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12294 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12295 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12296 }
12297 }
12298
12299 if (IOM_SUCCESS(rcStrict))
12300 {
12301 if (!fUpdateRipAlready)
12302 {
12303 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12304 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12305 }
12306
12307 /*
12308 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12309 * while booting Fedora 17 64-bit guest.
12310 *
12311 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12312 */
12313 if (fIOString)
12314 {
12315 /** @todo Single-step for INS/OUTS with REP prefix? */
12316 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12317 }
12318 else if ( !fDbgStepping
12319 && fGstStepping)
12320 {
12321 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12322 AssertRCReturn(rc, rc);
12323 }
12324
12325 /*
12326 * If any I/O breakpoints are armed, we need to check if one triggered
12327 * and take appropriate action.
12328 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12329 */
12330 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12331 AssertRCReturn(rc, rc);
12332
12333 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12334 * execution engines about whether hyper BPs and such are pending. */
12335 uint32_t const uDr7 = pMixedCtx->dr[7];
12336 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12337 && X86_DR7_ANY_RW_IO(uDr7)
12338 && (pMixedCtx->cr4 & X86_CR4_DE))
12339 || DBGFBpIsHwIoArmed(pVM)))
12340 {
12341 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12342
12343 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12344 VMMRZCallRing3Disable(pVCpu);
12345 HM_DISABLE_PREEMPT();
12346
12347 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12348
12349 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12350 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12351 {
12352 /* Raise #DB. */
12353 if (fIsGuestDbgActive)
12354 ASMSetDR6(pMixedCtx->dr[6]);
12355 if (pMixedCtx->dr[7] != uDr7)
12356 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12357
12358 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12359 }
12360 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12361 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12362 else if ( rcStrict2 != VINF_SUCCESS
12363 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12364 rcStrict = rcStrict2;
12365 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12366
12367 HM_RESTORE_PREEMPT();
12368 VMMRZCallRing3Enable(pVCpu);
12369 }
12370 }
12371
12372#ifdef VBOX_STRICT
12373 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12374 Assert(!fIOWrite);
12375 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12376 Assert(fIOWrite);
12377 else
12378 {
12379# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12380 * statuses, that the VMM device and some others may return. See
12381 * IOM_SUCCESS() for guidance. */
12382 AssertMsg( RT_FAILURE(rcStrict)
12383 || rcStrict == VINF_SUCCESS
12384 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12385 || rcStrict == VINF_EM_DBG_BREAKPOINT
12386 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12387 || rcStrict == VINF_EM_RAW_TO_R3
12388 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12389# endif
12390 }
12391#endif
12392 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12393 }
12394 else
12395 {
12396 /*
12397 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12398 */
12399 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12400 AssertRCReturn(rc2, rc2);
12401 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12402 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12403 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12404 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12405 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12406 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12407
12408 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12409 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12410
12411 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12412 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12413 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12414 }
12415 return rcStrict;
12416}
12417
12418
12419/**
12420 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12421 * VM-exit.
12422 */
12423HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12424{
12425 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12426
12427 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12428 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12429 AssertRCReturn(rc, rc);
12430 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12431 {
12432 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12433 AssertRCReturn(rc, rc);
12434 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12435 {
12436 uint32_t uErrCode;
12437 RTGCUINTPTR GCPtrFaultAddress;
12438 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12439 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12440 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12441 if (fErrorCodeValid)
12442 {
12443 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12444 AssertRCReturn(rc, rc);
12445 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12446 }
12447 else
12448 uErrCode = 0;
12449
12450 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12451 && uVector == X86_XCPT_PF)
12452 GCPtrFaultAddress = pMixedCtx->cr2;
12453 else
12454 GCPtrFaultAddress = 0;
12455
12456 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12457 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12458
12459 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12460 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12461 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12462 }
12463 }
12464
12465 /* Fall back to the interpreter to emulate the task-switch. */
12466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12467 return VERR_EM_INTERPRETER;
12468}
12469
12470
12471/**
12472 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12473 */
12474HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12475{
12476 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12477 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12478 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12479 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12480 AssertRCReturn(rc, rc);
12481 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12482 return VINF_EM_DBG_STEPPED;
12483}
12484
12485
12486/**
12487 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12488 */
12489HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12490{
12491 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12492
12493 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12494
12495 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12496 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12497 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12498 {
12499 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12500 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12501 {
12502 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12503 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12504 }
12505 }
12506 else
12507 {
12508 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12509 rcStrict1 = VINF_SUCCESS;
12510 return rcStrict1;
12511 }
12512
12513 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12514 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12515 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12516 AssertRCReturn(rc, rc);
12517
12518 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12519 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12520 VBOXSTRICTRC rcStrict2;
12521 switch (uAccessType)
12522 {
12523 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12524 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12525 {
12526 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12527 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12528 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12529
12530 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12531 GCPhys &= PAGE_BASE_GC_MASK;
12532 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12533 PVM pVM = pVCpu->CTX_SUFF(pVM);
12534 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12535 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12536
12537 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12538 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12539 CPUMCTX2CORE(pMixedCtx), GCPhys);
12540 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12541 if ( rcStrict2 == VINF_SUCCESS
12542 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12543 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12544 {
12545 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12546 | HM_CHANGED_GUEST_RSP
12547 | HM_CHANGED_GUEST_RFLAGS
12548 | HM_CHANGED_GUEST_APIC_TPR);
12549 rcStrict2 = VINF_SUCCESS;
12550 }
12551 break;
12552 }
12553
12554 default:
12555 Log4Func(("uAccessType=%#x\n", uAccessType));
12556 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12557 break;
12558 }
12559
12560 if (rcStrict2 != VINF_SUCCESS)
12561 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12562 return rcStrict2;
12563}
12564
12565
12566/**
12567 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12568 * VM-exit.
12569 */
12570HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12571{
12572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12573
12574 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12575 if (pVmxTransient->fWasGuestDebugStateActive)
12576 {
12577 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12578 HMVMX_UNEXPECTED_EXIT_RET(pVCpu, pVmxTransient);
12579 }
12580
12581 if ( !pVCpu->hm.s.fSingleInstruction
12582 && !pVmxTransient->fWasHyperDebugStateActive)
12583 {
12584 Assert(!DBGFIsStepping(pVCpu));
12585 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12586
12587 /* Don't intercept MOV DRx any more. */
12588 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12589 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12590 AssertRCReturn(rc, rc);
12591
12592 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12593 VMMRZCallRing3Disable(pVCpu);
12594 HM_DISABLE_PREEMPT();
12595
12596 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12597 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12598 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12599
12600 HM_RESTORE_PREEMPT();
12601 VMMRZCallRing3Enable(pVCpu);
12602
12603#ifdef VBOX_WITH_STATISTICS
12604 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12605 AssertRCReturn(rc, rc);
12606 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12607 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12608 else
12609 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12610#endif
12611 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12612 return VINF_SUCCESS;
12613 }
12614
12615 /*
12616 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12617 * Update the segment registers and DR7 from the CPU.
12618 */
12619 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12620 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12621 | CPUMCTX_EXTRN_DR7);
12622 AssertRCReturn(rc, rc);
12623 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12624
12625 PVM pVM = pVCpu->CTX_SUFF(pVM);
12626 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12627 {
12628 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12629 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12630 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12631 if (RT_SUCCESS(rc))
12632 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12633 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12634 }
12635 else
12636 {
12637 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12638 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12639 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12640 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12641 }
12642
12643 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12644 if (RT_SUCCESS(rc))
12645 {
12646 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12647 AssertRCReturn(rc2, rc2);
12648 return VINF_SUCCESS;
12649 }
12650 return rc;
12651}
12652
12653
12654/**
12655 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12656 * Conditional VM-exit.
12657 */
12658HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12659{
12660 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12661 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12662
12663 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12664 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12665 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12666 {
12667 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12668 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12669 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12670 {
12671 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12672 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12673 }
12674 }
12675 else
12676 {
12677 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12678 rcStrict1 = VINF_SUCCESS;
12679 return rcStrict1;
12680 }
12681
12682 /*
12683 * Get sufficent state and update the exit history entry.
12684 */
12685 RTGCPHYS GCPhys;
12686 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12687 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12688 AssertRCReturn(rc, rc);
12689
12690 VBOXSTRICTRC rcStrict;
12691 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12692 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12693 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12694 if (!pExitRec)
12695 {
12696 /*
12697 * If we succeed, resume guest execution.
12698 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12699 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12700 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12701 * weird case. See @bugref{6043}.
12702 */
12703 PVM pVM = pVCpu->CTX_SUFF(pVM);
12704 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12705 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12706 if ( rcStrict == VINF_SUCCESS
12707 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12708 || rcStrict == VERR_PAGE_NOT_PRESENT)
12709 {
12710 /* Successfully handled MMIO operation. */
12711 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12712 | HM_CHANGED_GUEST_RSP
12713 | HM_CHANGED_GUEST_RFLAGS
12714 | HM_CHANGED_GUEST_APIC_TPR);
12715 rcStrict = VINF_SUCCESS;
12716 }
12717 }
12718 else
12719 {
12720 /*
12721 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12722 */
12723 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12724 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12725 AssertRCReturn(rc2, rc2);
12726
12727 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12728 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12729
12730 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12731 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12732
12733 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12734 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12735 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12736 }
12737 return VBOXSTRICTRC_TODO(rcStrict);
12738}
12739
12740
12741/**
12742 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12743 * VM-exit.
12744 */
12745HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12746{
12747 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12748 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12749
12750 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12751 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12752 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12753 {
12754 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12755 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12756 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12757 }
12758 else
12759 {
12760 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12761 rcStrict1 = VINF_SUCCESS;
12762 return rcStrict1;
12763 }
12764
12765 RTGCPHYS GCPhys;
12766 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12767 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12768 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12769 AssertRCReturn(rc, rc);
12770
12771 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12772 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12773
12774 RTGCUINT uErrorCode = 0;
12775 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12776 uErrorCode |= X86_TRAP_PF_ID;
12777 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12778 uErrorCode |= X86_TRAP_PF_RW;
12779 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12780 uErrorCode |= X86_TRAP_PF_P;
12781
12782 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12783
12784 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12785 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12786
12787 /* Handle the pagefault trap for the nested shadow table. */
12788 PVM pVM = pVCpu->CTX_SUFF(pVM);
12789 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12790 TRPMResetTrap(pVCpu);
12791
12792 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12793 if ( rcStrict2 == VINF_SUCCESS
12794 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12795 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12796 {
12797 /* Successfully synced our nested page tables. */
12798 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12799 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12800 | HM_CHANGED_GUEST_RSP
12801 | HM_CHANGED_GUEST_RFLAGS);
12802 return VINF_SUCCESS;
12803 }
12804
12805 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12806 return rcStrict2;
12807}
12808
12809/** @} */
12810
12811/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12812/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12813/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12814
12815/** @name VM-exit exception handlers.
12816 * @{
12817 */
12818
12819/**
12820 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12821 */
12822static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12823{
12824 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12825 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12826
12827 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12828 AssertRCReturn(rc, rc);
12829
12830 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12831 {
12832 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12833 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12834
12835 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12836 * provides VM-exit instruction length. If this causes problem later,
12837 * disassemble the instruction like it's done on AMD-V. */
12838 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12839 AssertRCReturn(rc2, rc2);
12840 return rc;
12841 }
12842
12843 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12844 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12845 return rc;
12846}
12847
12848
12849/**
12850 * VM-exit exception handler for \#BP (Breakpoint exception).
12851 */
12852static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12853{
12854 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12856
12857 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12858 AssertRCReturn(rc, rc);
12859
12860 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12861 if (rc == VINF_EM_RAW_GUEST_TRAP)
12862 {
12863 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12864 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12865 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12866 AssertRCReturn(rc, rc);
12867
12868 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12869 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12870 }
12871
12872 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12873 return rc;
12874}
12875
12876
12877/**
12878 * VM-exit exception handler for \#AC (alignment check exception).
12879 */
12880static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12881{
12882 RT_NOREF_PV(pMixedCtx);
12883 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12884
12885 /*
12886 * Re-inject it. We'll detect any nesting before getting here.
12887 */
12888 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12889 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12890 AssertRCReturn(rc, rc);
12891 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12892
12893 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12894 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12895 return VINF_SUCCESS;
12896}
12897
12898
12899/**
12900 * VM-exit exception handler for \#DB (Debug exception).
12901 */
12902static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12903{
12904 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12905 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12906
12907 /*
12908 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12909 * for processing.
12910 */
12911 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12912
12913 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12914 uint64_t uDR6 = X86_DR6_INIT_VAL;
12915 uDR6 |= ( pVmxTransient->uExitQualification
12916 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12917
12918 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12919 Log6Func(("rc=%Rrc\n", rc));
12920 if (rc == VINF_EM_RAW_GUEST_TRAP)
12921 {
12922 /*
12923 * The exception was for the guest. Update DR6, DR7.GD and
12924 * IA32_DEBUGCTL.LBR before forwarding it.
12925 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12926 */
12927 VMMRZCallRing3Disable(pVCpu);
12928 HM_DISABLE_PREEMPT();
12929
12930 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12931 pMixedCtx->dr[6] |= uDR6;
12932 if (CPUMIsGuestDebugStateActive(pVCpu))
12933 ASMSetDR6(pMixedCtx->dr[6]);
12934
12935 HM_RESTORE_PREEMPT();
12936 VMMRZCallRing3Enable(pVCpu);
12937
12938 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12939 AssertRCReturn(rc, rc);
12940
12941 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12942 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12943
12944 /* Paranoia. */
12945 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12946 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12947
12948 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12949 AssertRCReturn(rc, rc);
12950
12951 /*
12952 * Raise #DB in the guest.
12953 *
12954 * It is important to reflect exactly what the VM-exit gave us (preserving the
12955 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12956 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12957 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12958 *
12959 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12960 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12961 */
12962 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12963 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12964 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12965 AssertRCReturn(rc, rc);
12966 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12967 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12968 return VINF_SUCCESS;
12969 }
12970
12971 /*
12972 * Not a guest trap, must be a hypervisor related debug event then.
12973 * Update DR6 in case someone is interested in it.
12974 */
12975 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12976 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12977 CPUMSetHyperDR6(pVCpu, uDR6);
12978
12979 return rc;
12980}
12981
12982/**
12983 * VM-exit exception handler for \#GP (General-protection exception).
12984 *
12985 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12986 */
12987static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12988{
12989 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12990 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12991
12992 int rc;
12993 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12994 { /* likely */ }
12995 else
12996 {
12997#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12998 Assert(pVCpu->hm.s.fUsingDebugLoop);
12999#endif
13000 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13001 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13002 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13003 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13004 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13005 AssertRCReturn(rc, rc);
13006 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13007 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13008 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13009 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13010 return rc;
13011 }
13012
13013 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13014 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13015
13016 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13017 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13018 AssertRCReturn(rc, rc);
13019
13020 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13021 uint32_t cbOp = 0;
13022 PVM pVM = pVCpu->CTX_SUFF(pVM);
13023 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13024 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13025 if (RT_SUCCESS(rc))
13026 {
13027 rc = VINF_SUCCESS;
13028 Assert(cbOp == pDis->cbInstr);
13029 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13030 switch (pDis->pCurInstr->uOpcode)
13031 {
13032 case OP_CLI:
13033 {
13034 pMixedCtx->eflags.Bits.u1IF = 0;
13035 pMixedCtx->eflags.Bits.u1RF = 0;
13036 pMixedCtx->rip += pDis->cbInstr;
13037 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13038 if ( !fDbgStepping
13039 && pMixedCtx->eflags.Bits.u1TF)
13040 {
13041 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13042 AssertRCReturn(rc, rc);
13043 }
13044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13045 break;
13046 }
13047
13048 case OP_STI:
13049 {
13050 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13051 pMixedCtx->eflags.Bits.u1IF = 1;
13052 pMixedCtx->eflags.Bits.u1RF = 0;
13053 pMixedCtx->rip += pDis->cbInstr;
13054 if (!fOldIF)
13055 {
13056 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13057 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13058 }
13059 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13060 if ( !fDbgStepping
13061 && pMixedCtx->eflags.Bits.u1TF)
13062 {
13063 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13064 AssertRCReturn(rc, rc);
13065 }
13066 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13067 break;
13068 }
13069
13070 case OP_HLT:
13071 {
13072 rc = VINF_EM_HALT;
13073 pMixedCtx->rip += pDis->cbInstr;
13074 pMixedCtx->eflags.Bits.u1RF = 0;
13075 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13076 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13077 break;
13078 }
13079
13080 case OP_POPF:
13081 {
13082 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13083 uint32_t cbParm;
13084 uint32_t uMask;
13085 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13086 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13087 {
13088 cbParm = 4;
13089 uMask = 0xffffffff;
13090 }
13091 else
13092 {
13093 cbParm = 2;
13094 uMask = 0xffff;
13095 }
13096
13097 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13098 RTGCPTR GCPtrStack = 0;
13099 X86EFLAGS Eflags;
13100 Eflags.u32 = 0;
13101 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13102 &GCPtrStack);
13103 if (RT_SUCCESS(rc))
13104 {
13105 Assert(sizeof(Eflags.u32) >= cbParm);
13106 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13107 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13108 }
13109 if (RT_FAILURE(rc))
13110 {
13111 rc = VERR_EM_INTERPRETER;
13112 break;
13113 }
13114 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13115 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13116 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13117 pMixedCtx->esp += cbParm;
13118 pMixedCtx->esp &= uMask;
13119 pMixedCtx->rip += pDis->cbInstr;
13120 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13121 | HM_CHANGED_GUEST_RSP
13122 | HM_CHANGED_GUEST_RFLAGS);
13123 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13124 POPF restores EFLAGS.TF. */
13125 if ( !fDbgStepping
13126 && fGstStepping)
13127 {
13128 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13129 AssertRCReturn(rc, rc);
13130 }
13131 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13132 break;
13133 }
13134
13135 case OP_PUSHF:
13136 {
13137 uint32_t cbParm;
13138 uint32_t uMask;
13139 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13140 {
13141 cbParm = 4;
13142 uMask = 0xffffffff;
13143 }
13144 else
13145 {
13146 cbParm = 2;
13147 uMask = 0xffff;
13148 }
13149
13150 /* Get the stack pointer & push the contents of eflags onto the stack. */
13151 RTGCPTR GCPtrStack = 0;
13152 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13153 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13154 if (RT_FAILURE(rc))
13155 {
13156 rc = VERR_EM_INTERPRETER;
13157 break;
13158 }
13159 X86EFLAGS Eflags = pMixedCtx->eflags;
13160 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13161 Eflags.Bits.u1RF = 0;
13162 Eflags.Bits.u1VM = 0;
13163
13164 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13165 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13166 {
13167 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13168 rc = VERR_EM_INTERPRETER;
13169 break;
13170 }
13171 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13172 pMixedCtx->esp -= cbParm;
13173 pMixedCtx->esp &= uMask;
13174 pMixedCtx->rip += pDis->cbInstr;
13175 pMixedCtx->eflags.Bits.u1RF = 0;
13176 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13177 | HM_CHANGED_GUEST_RSP
13178 | HM_CHANGED_GUEST_RFLAGS);
13179 if ( !fDbgStepping
13180 && pMixedCtx->eflags.Bits.u1TF)
13181 {
13182 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13183 AssertRCReturn(rc, rc);
13184 }
13185 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13186 break;
13187 }
13188
13189 case OP_IRET:
13190 {
13191 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13192 * instruction reference. */
13193 RTGCPTR GCPtrStack = 0;
13194 uint32_t uMask = 0xffff;
13195 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13196 uint16_t aIretFrame[3];
13197 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13198 {
13199 rc = VERR_EM_INTERPRETER;
13200 break;
13201 }
13202 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13203 &GCPtrStack);
13204 if (RT_SUCCESS(rc))
13205 {
13206 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13207 PGMACCESSORIGIN_HM));
13208 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13209 }
13210 if (RT_FAILURE(rc))
13211 {
13212 rc = VERR_EM_INTERPRETER;
13213 break;
13214 }
13215 pMixedCtx->eip = 0;
13216 pMixedCtx->ip = aIretFrame[0];
13217 pMixedCtx->cs.Sel = aIretFrame[1];
13218 pMixedCtx->cs.ValidSel = aIretFrame[1];
13219 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13220 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13221 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13222 pMixedCtx->sp += sizeof(aIretFrame);
13223 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13224 | HM_CHANGED_GUEST_CS
13225 | HM_CHANGED_GUEST_RSP
13226 | HM_CHANGED_GUEST_RFLAGS);
13227 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13228 if ( !fDbgStepping
13229 && fGstStepping)
13230 {
13231 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13232 AssertRCReturn(rc, rc);
13233 }
13234 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13235 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13236 break;
13237 }
13238
13239 case OP_INT:
13240 {
13241 uint16_t uVector = pDis->Param1.uValue & 0xff;
13242 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13243 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13244 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13245 break;
13246 }
13247
13248 case OP_INTO:
13249 {
13250 if (pMixedCtx->eflags.Bits.u1OF)
13251 {
13252 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13253 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13254 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13255 }
13256 else
13257 {
13258 pMixedCtx->eflags.Bits.u1RF = 0;
13259 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13260 }
13261 break;
13262 }
13263
13264 default:
13265 {
13266 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13267 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13268 EMCODETYPE_SUPERVISOR);
13269 rc = VBOXSTRICTRC_VAL(rc2);
13270 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13271 /** @todo We have to set pending-debug exceptions here when the guest is
13272 * single-stepping depending on the instruction that was interpreted. */
13273 Log4Func(("#GP rc=%Rrc\n", rc));
13274 break;
13275 }
13276 }
13277 }
13278 else
13279 rc = VERR_EM_INTERPRETER;
13280
13281 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13282 ("#GP Unexpected rc=%Rrc\n", rc));
13283 return rc;
13284}
13285
13286
13287/**
13288 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13289 * the exception reported in the VMX transient structure back into the VM.
13290 *
13291 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13292 * up-to-date.
13293 */
13294static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13295{
13296 RT_NOREF_PV(pMixedCtx);
13297 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13298#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13299 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13300 ("uVector=%#x u32XcptBitmap=%#X32\n",
13301 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13302#endif
13303
13304 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13305 hmR0VmxCheckExitDueToEventDelivery(). */
13306 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13307 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13308 AssertRCReturn(rc, rc);
13309 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13310
13311#ifdef DEBUG_ramshankar
13312 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13313 | CPUMCTX_EXTRN_RIP);
13314 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13315 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13316#endif
13317
13318 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13319 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13320 return VINF_SUCCESS;
13321}
13322
13323
13324/**
13325 * VM-exit exception handler for \#PF (Page-fault exception).
13326 */
13327static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13328{
13329 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13330 PVM pVM = pVCpu->CTX_SUFF(pVM);
13331 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13332 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13333 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13334 AssertRCReturn(rc, rc);
13335
13336 if (!pVM->hm.s.fNestedPaging)
13337 { /* likely */ }
13338 else
13339 {
13340#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13341 Assert(pVCpu->hm.s.fUsingDebugLoop);
13342#endif
13343 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13344 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13345 {
13346 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13347 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13348 }
13349 else
13350 {
13351 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13352 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13353 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13354 }
13355 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13356 return rc;
13357 }
13358
13359 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13360 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13361 if (pVmxTransient->fVectoringPF)
13362 {
13363 Assert(pVCpu->hm.s.Event.fPending);
13364 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13365 }
13366
13367 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13368 AssertRCReturn(rc, rc);
13369
13370 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13371 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13372
13373 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13374 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13375 (RTGCPTR)pVmxTransient->uExitQualification);
13376
13377 Log4Func(("#PF: rc=%Rrc\n", rc));
13378 if (rc == VINF_SUCCESS)
13379 {
13380 /*
13381 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13382 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13383 */
13384 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13385 TRPMResetTrap(pVCpu);
13386 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13387 return rc;
13388 }
13389
13390 if (rc == VINF_EM_RAW_GUEST_TRAP)
13391 {
13392 if (!pVmxTransient->fVectoringDoublePF)
13393 {
13394 /* It's a guest page fault and needs to be reflected to the guest. */
13395 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13396 TRPMResetTrap(pVCpu);
13397 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13398 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13399 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13400 }
13401 else
13402 {
13403 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13404 TRPMResetTrap(pVCpu);
13405 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13406 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13407 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13408 }
13409
13410 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13411 return VINF_SUCCESS;
13412 }
13413
13414 TRPMResetTrap(pVCpu);
13415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13416 return rc;
13417}
13418
13419/** @} */
13420
Note: See TracBrowser for help on using the repository browser.

© 2025 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette