VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72850

Last change on this file since 72850 was 72850, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Spaces, remove unused macro.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 575.5 KB
Line 
1/* $Id: HMVMXR0.cpp 72850 2018-07-04 05:33:06Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/** Maximum VM-instruction error number. */
134#define HMVMX_INSTR_ERROR_MAX 28
135
136/** Profiling macro. */
137#ifdef HM_PROFILE_EXIT_DISPATCH
138# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
139# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
140#else
141# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
142# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
143#endif
144
145/** Assert that preemption is disabled or covered by thread-context hooks. */
146#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
147 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD))
148
149/** Assert that we haven't migrated CPUs when thread-context hooks are not
150 * used. */
151#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
152 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
153 ("Illegal migration! Entered on CPU %u Current %u\n", \
154 pVCpu->hm.s.idEnteredCpu, RTMpCpuId()))
155
156/** Asserts that the given CPUMCTX_EXTRN_XXX bits are present in the guest-CPU
157 * context. */
158#define HMVMX_CPUMCTX_ASSERT(a_pVCpu, a_fExtrnMbz) AssertMsg(!((a_pVCpu)->cpum.GstCtx.fExtrn & (a_fExtrnMbz)), \
159 ("fExtrn=%#RX64 fExtrnMbz=%#RX64\n", \
160 (a_pVCpu)->cpum.GstCtx.fExtrn, (a_fExtrnMbz)))
161
162
163/** Helper macro for VM-exit handlers called unexpectedly. */
164#define HMVMX_RETURN_UNEXPECTED_EXIT() \
165 do { \
166 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
167 return VERR_VMX_UNEXPECTED_EXIT; \
168 } while (0)
169
170/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
171#ifdef VMX_USE_CACHED_VMCS_ACCESSES
172# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
173 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
174 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
175#else
176# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
177 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
178 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
179#endif
180
181/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
182# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
183 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
184 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
185
186
187/*********************************************************************************************************************************
188* Structures and Typedefs *
189*********************************************************************************************************************************/
190/**
191 * VMX transient state.
192 *
193 * A state structure for holding miscellaneous information across
194 * VMX non-root operation and restored after the transition.
195 */
196typedef struct VMXTRANSIENT
197{
198 /** The host's rflags/eflags. */
199 RTCCUINTREG fEFlags;
200#if HC_ARCH_BITS == 32
201 uint32_t u32Alignment0;
202#endif
203 /** The guest's TPR value used for TPR shadowing. */
204 uint8_t u8GuestTpr;
205 /** Alignment. */
206 uint8_t abAlignment0[7];
207
208 /** The basic VM-exit reason. */
209 uint16_t uExitReason;
210 /** Alignment. */
211 uint16_t u16Alignment0;
212 /** The VM-exit interruption error code. */
213 uint32_t uExitIntErrorCode;
214 /** The VM-exit exit code qualification. */
215 uint64_t uExitQualification;
216
217 /** The VM-exit interruption-information field. */
218 uint32_t uExitIntInfo;
219 /** The VM-exit instruction-length field. */
220 uint32_t cbInstr;
221 /** The VM-exit instruction-information field. */
222 union
223 {
224 /** Plain unsigned int representation. */
225 uint32_t u;
226 /** INS and OUTS information. */
227 struct
228 {
229 uint32_t u7Reserved0 : 7;
230 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
231 uint32_t u3AddrSize : 3;
232 uint32_t u5Reserved1 : 5;
233 /** The segment register (X86_SREG_XXX). */
234 uint32_t iSegReg : 3;
235 uint32_t uReserved2 : 14;
236 } StrIo;
237 /** INVEPT, INVVPID, INVPCID information. */
238 struct
239 {
240 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
241 uint32_t u2Scaling : 2;
242 uint32_t u5Reserved0 : 5;
243 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
244 uint32_t u3AddrSize : 3;
245 uint32_t u1Reserved0 : 1;
246 uint32_t u4Reserved0 : 4;
247 /** The segment register (X86_SREG_XXX). */
248 uint32_t iSegReg : 3;
249 /** The index register (X86_GREG_XXX). */
250 uint32_t iIdxReg : 4;
251 /** Set if index register is invalid. */
252 uint32_t fIdxRegValid : 1;
253 /** The base register (X86_GREG_XXX). */
254 uint32_t iBaseReg : 4;
255 /** Set if base register is invalid. */
256 uint32_t fBaseRegValid : 1;
257 /** Register 2 (X86_GREG_XXX). */
258 uint32_t iReg2 : 4;
259 } Inv;
260 } ExitInstrInfo;
261 /** Whether the VM-entry failed or not. */
262 bool fVMEntryFailed;
263 /** Alignment. */
264 uint8_t abAlignment1[3];
265
266 /** The VM-entry interruption-information field. */
267 uint32_t uEntryIntInfo;
268 /** The VM-entry exception error code field. */
269 uint32_t uEntryXcptErrorCode;
270 /** The VM-entry instruction length field. */
271 uint32_t cbEntryInstr;
272
273 /** IDT-vectoring information field. */
274 uint32_t uIdtVectoringInfo;
275 /** IDT-vectoring error code. */
276 uint32_t uIdtVectoringErrorCode;
277
278 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
279 uint32_t fVmcsFieldsRead;
280
281 /** Whether the guest debug state was active at the time of VM-exit. */
282 bool fWasGuestDebugStateActive;
283 /** Whether the hyper debug state was active at the time of VM-exit. */
284 bool fWasHyperDebugStateActive;
285 /** Whether TSC-offsetting should be setup before VM-entry. */
286 bool fUpdateTscOffsettingAndPreemptTimer;
287 /** Whether the VM-exit was caused by a page-fault during delivery of a
288 * contributory exception or a page-fault. */
289 bool fVectoringDoublePF;
290 /** Whether the VM-exit was caused by a page-fault during delivery of an
291 * external interrupt or NMI. */
292 bool fVectoringPF;
293} VMXTRANSIENT;
294AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
296AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
297AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
298AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
299/** Pointer to VMX transient state. */
300typedef VMXTRANSIENT *PVMXTRANSIENT;
301
302
303/**
304 * MSR-bitmap read permissions.
305 */
306typedef enum VMXMSREXITREAD
307{
308 /** Reading this MSR causes a VM-exit. */
309 VMXMSREXIT_INTERCEPT_READ = 0xb,
310 /** Reading this MSR does not cause a VM-exit. */
311 VMXMSREXIT_PASSTHRU_READ
312} VMXMSREXITREAD;
313/** Pointer to MSR-bitmap read permissions. */
314typedef VMXMSREXITREAD* PVMXMSREXITREAD;
315
316/**
317 * MSR-bitmap write permissions.
318 */
319typedef enum VMXMSREXITWRITE
320{
321 /** Writing to this MSR causes a VM-exit. */
322 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
323 /** Writing to this MSR does not cause a VM-exit. */
324 VMXMSREXIT_PASSTHRU_WRITE
325} VMXMSREXITWRITE;
326/** Pointer to MSR-bitmap write permissions. */
327typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
328
329
330/**
331 * VMX VM-exit handler.
332 *
333 * @returns Strict VBox status code (i.e. informational status codes too).
334 * @param pVCpu The cross context virtual CPU structure.
335 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
336 * out-of-sync. Make sure to update the required
337 * fields before using them.
338 * @param pVmxTransient Pointer to the VMX-transient structure.
339 */
340#ifndef HMVMX_USE_FUNCTION_TABLE
341typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342#else
343typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
344/** Pointer to VM-exit handler. */
345typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
346#endif
347
348/**
349 * VMX VM-exit handler, non-strict status code.
350 *
351 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
352 *
353 * @returns VBox status code, no informational status code returned.
354 * @param pVCpu The cross context virtual CPU structure.
355 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
356 * out-of-sync. Make sure to update the required
357 * fields before using them.
358 * @param pVmxTransient Pointer to the VMX-transient structure.
359 *
360 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
361 * use of that status code will be replaced with VINF_EM_SOMETHING
362 * later when switching over to IEM.
363 */
364#ifndef HMVMX_USE_FUNCTION_TABLE
365typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
366#else
367typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
368#endif
369
370
371/*********************************************************************************************************************************
372* Internal Functions *
373*********************************************************************************************************************************/
374static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
375static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
376static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
377static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
378static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
379 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
380#if HC_ARCH_BITS == 32
381static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
382#endif
383#ifndef HMVMX_USE_FUNCTION_TABLE
384DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
385# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
386# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
387#else
388# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
389# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
390#endif
391
392
393/** @name VM-exit handlers.
394 * @{
395 */
396static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
397static FNVMXEXITHANDLER hmR0VmxExitExtInt;
398static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
405static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
406static FNVMXEXITHANDLER hmR0VmxExitCpuid;
407static FNVMXEXITHANDLER hmR0VmxExitGetsec;
408static FNVMXEXITHANDLER hmR0VmxExitHlt;
409static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
410static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
411static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
412static FNVMXEXITHANDLER hmR0VmxExitVmcall;
413static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
414static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
415static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
416static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
417static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
418static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
419static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
420static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
422static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
424static FNVMXEXITHANDLER hmR0VmxExitMwait;
425static FNVMXEXITHANDLER hmR0VmxExitMtf;
426static FNVMXEXITHANDLER hmR0VmxExitMonitor;
427static FNVMXEXITHANDLER hmR0VmxExitPause;
428static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
429static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
430static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
431static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
432static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
433static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
434static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
435static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
436static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
437static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
438static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
439static FNVMXEXITHANDLER hmR0VmxExitRdrand;
440static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
441/** @} */
442
443static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
450static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
451
452
453/*********************************************************************************************************************************
454* Global Variables *
455*********************************************************************************************************************************/
456#ifdef HMVMX_USE_FUNCTION_TABLE
457
458/**
459 * VMX_EXIT dispatch table.
460 */
461static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
462{
463 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
464 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
465 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
466 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
467 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
468 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
469 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
470 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
471 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
472 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
473 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
474 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
475 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
476 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
477 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
478 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
479 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
480 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
481 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
482 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
483 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
484 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
485 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
486 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
487 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
488 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
489 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
490 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
491 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
492 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
493 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
494 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
495 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
496 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
497 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
498 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
499 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
500 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
501 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
502 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
503 /* 40 UNDEFINED */ hmR0VmxExitPause,
504 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
505 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
506 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
507 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
508 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
509 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
510 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
511 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
512 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
513 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
514 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
515 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
516 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
517 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
518 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
519 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
520 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
521 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
522 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
523 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
524 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
525 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
526 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
527 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
528};
529#endif /* HMVMX_USE_FUNCTION_TABLE */
530
531#ifdef VBOX_STRICT
532static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
533{
534 /* 0 */ "(Not Used)",
535 /* 1 */ "VMCALL executed in VMX root operation.",
536 /* 2 */ "VMCLEAR with invalid physical address.",
537 /* 3 */ "VMCLEAR with VMXON pointer.",
538 /* 4 */ "VMLAUNCH with non-clear VMCS.",
539 /* 5 */ "VMRESUME with non-launched VMCS.",
540 /* 6 */ "VMRESUME after VMXOFF",
541 /* 7 */ "VM-entry with invalid control fields.",
542 /* 8 */ "VM-entry with invalid host state fields.",
543 /* 9 */ "VMPTRLD with invalid physical address.",
544 /* 10 */ "VMPTRLD with VMXON pointer.",
545 /* 11 */ "VMPTRLD with incorrect revision identifier.",
546 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
547 /* 13 */ "VMWRITE to read-only VMCS component.",
548 /* 14 */ "(Not Used)",
549 /* 15 */ "VMXON executed in VMX root operation.",
550 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
551 /* 17 */ "VM-entry with non-launched executing VMCS.",
552 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
553 /* 19 */ "VMCALL with non-clear VMCS.",
554 /* 20 */ "VMCALL with invalid VM-exit control fields.",
555 /* 21 */ "(Not Used)",
556 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
557 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
558 /* 24 */ "VMCALL with invalid SMM-monitor features.",
559 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
560 /* 26 */ "VM-entry with events blocked by MOV SS.",
561 /* 27 */ "(Not Used)",
562 /* 28 */ "Invalid operand to INVEPT/INVVPID."
563};
564#endif /* VBOX_STRICT */
565
566
567
568/**
569 * Updates the VM's last error record.
570 *
571 * If there was a VMX instruction error, reads the error data from the VMCS and
572 * updates VCPU's last error record as well.
573 *
574 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
575 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
576 * VERR_VMX_INVALID_VMCS_FIELD.
577 * @param rc The error code.
578 */
579static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
580{
581 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
582 || rc == VERR_VMX_UNABLE_TO_START_VM)
583 {
584 AssertPtrReturnVoid(pVCpu);
585 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
586 }
587 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
588}
589
590
591/**
592 * Reads the VM-entry interruption-information field from the VMCS into the VMX
593 * transient structure.
594 *
595 * @returns VBox status code.
596 * @param pVmxTransient Pointer to the VMX transient structure.
597 *
598 * @remarks No-long-jump zone!!!
599 */
600DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
601{
602 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
603 AssertRCReturn(rc, rc);
604 return VINF_SUCCESS;
605}
606
607#ifdef VBOX_STRICT
608/**
609 * Reads the VM-entry exception error code field from the VMCS into
610 * the VMX transient structure.
611 *
612 * @returns VBox status code.
613 * @param pVmxTransient Pointer to the VMX transient structure.
614 *
615 * @remarks No-long-jump zone!!!
616 */
617DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
618{
619 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
620 AssertRCReturn(rc, rc);
621 return VINF_SUCCESS;
622}
623
624
625/**
626 * Reads the VM-entry exception error code field from the VMCS into
627 * the VMX transient structure.
628 *
629 * @returns VBox status code.
630 * @param pVmxTransient Pointer to the VMX transient structure.
631 *
632 * @remarks No-long-jump zone!!!
633 */
634DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
635{
636 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
637 AssertRCReturn(rc, rc);
638 return VINF_SUCCESS;
639}
640#endif /* VBOX_STRICT */
641
642
643/**
644 * Reads the VM-exit interruption-information field from the VMCS into the VMX
645 * transient structure.
646 *
647 * @returns VBox status code.
648 * @param pVmxTransient Pointer to the VMX transient structure.
649 */
650DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
651{
652 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
653 {
654 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
655 AssertRCReturn(rc,rc);
656 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
657 }
658 return VINF_SUCCESS;
659}
660
661
662/**
663 * Reads the VM-exit interruption error code from the VMCS into the VMX
664 * transient structure.
665 *
666 * @returns VBox status code.
667 * @param pVmxTransient Pointer to the VMX transient structure.
668 */
669DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
670{
671 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
672 {
673 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
674 AssertRCReturn(rc, rc);
675 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
676 }
677 return VINF_SUCCESS;
678}
679
680
681/**
682 * Reads the VM-exit instruction length field from the VMCS into the VMX
683 * transient structure.
684 *
685 * @returns VBox status code.
686 * @param pVmxTransient Pointer to the VMX transient structure.
687 */
688DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
689{
690 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
691 {
692 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
693 AssertRCReturn(rc, rc);
694 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
695 }
696 return VINF_SUCCESS;
697}
698
699
700/**
701 * Reads the VM-exit instruction-information field from the VMCS into
702 * the VMX transient structure.
703 *
704 * @returns VBox status code.
705 * @param pVmxTransient Pointer to the VMX transient structure.
706 */
707DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
708{
709 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
710 {
711 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
712 AssertRCReturn(rc, rc);
713 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
714 }
715 return VINF_SUCCESS;
716}
717
718
719/**
720 * Reads the exit code qualification from the VMCS into the VMX transient
721 * structure.
722 *
723 * @returns VBox status code.
724 * @param pVCpu The cross context virtual CPU structure of the
725 * calling EMT. (Required for the VMCS cache case.)
726 * @param pVmxTransient Pointer to the VMX transient structure.
727 */
728DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
729{
730 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
731 {
732 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
733 AssertRCReturn(rc, rc);
734 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
735 }
736 return VINF_SUCCESS;
737}
738
739
740/**
741 * Reads the IDT-vectoring information field from the VMCS into the VMX
742 * transient structure.
743 *
744 * @returns VBox status code.
745 * @param pVmxTransient Pointer to the VMX transient structure.
746 *
747 * @remarks No-long-jump zone!!!
748 */
749DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
750{
751 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
752 {
753 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
754 AssertRCReturn(rc, rc);
755 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
756 }
757 return VINF_SUCCESS;
758}
759
760
761/**
762 * Reads the IDT-vectoring error code from the VMCS into the VMX
763 * transient structure.
764 *
765 * @returns VBox status code.
766 * @param pVmxTransient Pointer to the VMX transient structure.
767 */
768DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
769{
770 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
771 {
772 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
773 AssertRCReturn(rc, rc);
774 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
775 }
776 return VINF_SUCCESS;
777}
778
779
780/**
781 * Enters VMX root mode operation on the current CPU.
782 *
783 * @returns VBox status code.
784 * @param pVM The cross context VM structure. Can be
785 * NULL, after a resume.
786 * @param HCPhysCpuPage Physical address of the VMXON region.
787 * @param pvCpuPage Pointer to the VMXON region.
788 */
789static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
790{
791 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
792 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
793 Assert(pvCpuPage);
794 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
795
796 if (pVM)
797 {
798 /* Write the VMCS revision dword to the VMXON region. */
799 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
800 }
801
802 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
803 RTCCUINTREG fEFlags = ASMIntDisableFlags();
804
805 /* Enable the VMX bit in CR4 if necessary. */
806 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
807
808 /* Enter VMX root mode. */
809 int rc = VMXEnable(HCPhysCpuPage);
810 if (RT_FAILURE(rc))
811 {
812 if (!(uOldCr4 & X86_CR4_VMXE))
813 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
814
815 if (pVM)
816 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
817 }
818
819 /* Restore interrupts. */
820 ASMSetFlags(fEFlags);
821 return rc;
822}
823
824
825/**
826 * Exits VMX root mode operation on the current CPU.
827 *
828 * @returns VBox status code.
829 */
830static int hmR0VmxLeaveRootMode(void)
831{
832 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
833
834 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
835 RTCCUINTREG fEFlags = ASMIntDisableFlags();
836
837 /* If we're for some reason not in VMX root mode, then don't leave it. */
838 RTCCUINTREG uHostCR4 = ASMGetCR4();
839
840 int rc;
841 if (uHostCR4 & X86_CR4_VMXE)
842 {
843 /* Exit VMX root mode and clear the VMX bit in CR4. */
844 VMXDisable();
845 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
846 rc = VINF_SUCCESS;
847 }
848 else
849 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
850
851 /* Restore interrupts. */
852 ASMSetFlags(fEFlags);
853 return rc;
854}
855
856
857/**
858 * Allocates and maps one physically contiguous page. The allocated page is
859 * zero'd out. (Used by various VT-x structures).
860 *
861 * @returns IPRT status code.
862 * @param pMemObj Pointer to the ring-0 memory object.
863 * @param ppVirt Where to store the virtual address of the
864 * allocation.
865 * @param pHCPhys Where to store the physical address of the
866 * allocation.
867 */
868static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
869{
870 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
871 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
872 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
873
874 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
875 if (RT_FAILURE(rc))
876 return rc;
877 *ppVirt = RTR0MemObjAddress(*pMemObj);
878 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
879 ASMMemZero32(*ppVirt, PAGE_SIZE);
880 return VINF_SUCCESS;
881}
882
883
884/**
885 * Frees and unmaps an allocated physical page.
886 *
887 * @param pMemObj Pointer to the ring-0 memory object.
888 * @param ppVirt Where to re-initialize the virtual address of
889 * allocation as 0.
890 * @param pHCPhys Where to re-initialize the physical address of the
891 * allocation as 0.
892 */
893static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
894{
895 AssertPtr(pMemObj);
896 AssertPtr(ppVirt);
897 AssertPtr(pHCPhys);
898 if (*pMemObj != NIL_RTR0MEMOBJ)
899 {
900 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
901 AssertRC(rc);
902 *pMemObj = NIL_RTR0MEMOBJ;
903 *ppVirt = 0;
904 *pHCPhys = 0;
905 }
906}
907
908
909/**
910 * Worker function to free VT-x related structures.
911 *
912 * @returns IPRT status code.
913 * @param pVM The cross context VM structure.
914 */
915static void hmR0VmxStructsFree(PVM pVM)
916{
917 for (VMCPUID i = 0; i < pVM->cCpus; i++)
918 {
919 PVMCPU pVCpu = &pVM->aCpus[i];
920 AssertPtr(pVCpu);
921
922 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
923 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
924
925 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
927
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
929 }
930
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
932#ifdef VBOX_WITH_CRASHDUMP_MAGIC
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
934#endif
935}
936
937
938/**
939 * Worker function to allocate VT-x related VM structures.
940 *
941 * @returns IPRT status code.
942 * @param pVM The cross context VM structure.
943 */
944static int hmR0VmxStructsAlloc(PVM pVM)
945{
946 /*
947 * Initialize members up-front so we can cleanup properly on allocation failure.
948 */
949#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
950 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
951 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
952 pVM->hm.s.vmx.HCPhys##a_Name = 0;
953
954#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
955 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
956 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
957 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
958
959#ifdef VBOX_WITH_CRASHDUMP_MAGIC
960 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
961#endif
962 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
963
964 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
965 for (VMCPUID i = 0; i < pVM->cCpus; i++)
966 {
967 PVMCPU pVCpu = &pVM->aCpus[i];
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
972 }
973#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
974#undef VMXLOCAL_INIT_VM_MEMOBJ
975
976 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
977 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
978 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
979 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
980
981 /*
982 * Allocate all the VT-x structures.
983 */
984 int rc = VINF_SUCCESS;
985#ifdef VBOX_WITH_CRASHDUMP_MAGIC
986 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
987 if (RT_FAILURE(rc))
988 goto cleanup;
989 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
990 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
991#endif
992
993 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
994 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
995 {
996 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
997 &pVM->hm.s.vmx.HCPhysApicAccess);
998 if (RT_FAILURE(rc))
999 goto cleanup;
1000 }
1001
1002 /*
1003 * Initialize per-VCPU VT-x structures.
1004 */
1005 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1006 {
1007 PVMCPU pVCpu = &pVM->aCpus[i];
1008 AssertPtr(pVCpu);
1009
1010 /* Allocate the VM control structure (VMCS). */
1011 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1012 if (RT_FAILURE(rc))
1013 goto cleanup;
1014
1015 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1016 if ( PDMHasApic(pVM)
1017 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1018 {
1019 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1020 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1021 if (RT_FAILURE(rc))
1022 goto cleanup;
1023 }
1024
1025 /*
1026 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1027 * transparent accesses of specific MSRs.
1028 *
1029 * If the condition for enabling MSR bitmaps changes here, don't forget to
1030 * update HMAreMsrBitmapsAvailable().
1031 */
1032 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1033 {
1034 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1035 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1036 if (RT_FAILURE(rc))
1037 goto cleanup;
1038 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1039 }
1040
1041 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1042 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1043 if (RT_FAILURE(rc))
1044 goto cleanup;
1045
1046 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1047 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1048 if (RT_FAILURE(rc))
1049 goto cleanup;
1050 }
1051
1052 return VINF_SUCCESS;
1053
1054cleanup:
1055 hmR0VmxStructsFree(pVM);
1056 return rc;
1057}
1058
1059
1060/**
1061 * Does global VT-x initialization (called during module initialization).
1062 *
1063 * @returns VBox status code.
1064 */
1065VMMR0DECL(int) VMXR0GlobalInit(void)
1066{
1067#ifdef HMVMX_USE_FUNCTION_TABLE
1068 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1069# ifdef VBOX_STRICT
1070 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1071 Assert(g_apfnVMExitHandlers[i]);
1072# endif
1073#endif
1074 return VINF_SUCCESS;
1075}
1076
1077
1078/**
1079 * Does global VT-x termination (called during module termination).
1080 */
1081VMMR0DECL(void) VMXR0GlobalTerm()
1082{
1083 /* Nothing to do currently. */
1084}
1085
1086
1087/**
1088 * Sets up and activates VT-x on the current CPU.
1089 *
1090 * @returns VBox status code.
1091 * @param pHostCpu Pointer to the global CPU info struct.
1092 * @param pVM The cross context VM structure. Can be
1093 * NULL after a host resume operation.
1094 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1095 * fEnabledByHost is @c true).
1096 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1097 * @a fEnabledByHost is @c true).
1098 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1099 * enable VT-x on the host.
1100 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1101 */
1102VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1103 void *pvMsrs)
1104{
1105 Assert(pHostCpu);
1106 Assert(pvMsrs);
1107 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1108
1109 /* Enable VT-x if it's not already enabled by the host. */
1110 if (!fEnabledByHost)
1111 {
1112 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1113 if (RT_FAILURE(rc))
1114 return rc;
1115 }
1116
1117 /*
1118 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1119 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1120 * invalidated when flushing by VPID.
1121 */
1122 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1123 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1124 {
1125 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1126 pHostCpu->fFlushAsidBeforeUse = false;
1127 }
1128 else
1129 pHostCpu->fFlushAsidBeforeUse = true;
1130
1131 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1132 ++pHostCpu->cTlbFlushes;
1133
1134 return VINF_SUCCESS;
1135}
1136
1137
1138/**
1139 * Deactivates VT-x on the current CPU.
1140 *
1141 * @returns VBox status code.
1142 * @param pHostCpu Pointer to the global CPU info struct.
1143 * @param pvCpuPage Pointer to the VMXON region.
1144 * @param HCPhysCpuPage Physical address of the VMXON region.
1145 *
1146 * @remarks This function should never be called when SUPR0EnableVTx() or
1147 * similar was used to enable VT-x on the host.
1148 */
1149VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1150{
1151 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1152
1153 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1154 return hmR0VmxLeaveRootMode();
1155}
1156
1157
1158/**
1159 * Sets the permission bits for the specified MSR in the MSR bitmap.
1160 *
1161 * @param pVCpu The cross context virtual CPU structure.
1162 * @param uMsr The MSR value.
1163 * @param enmRead Whether reading this MSR causes a VM-exit.
1164 * @param enmWrite Whether writing this MSR causes a VM-exit.
1165 */
1166static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1167{
1168 int32_t iBit;
1169 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1170
1171 /*
1172 * Layout:
1173 * 0x000 - 0x3ff - Low MSR read bits
1174 * 0x400 - 0x7ff - High MSR read bits
1175 * 0x800 - 0xbff - Low MSR write bits
1176 * 0xc00 - 0xfff - High MSR write bits
1177 */
1178 if (uMsr <= 0x00001FFF)
1179 iBit = uMsr;
1180 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1181 {
1182 iBit = uMsr - UINT32_C(0xC0000000);
1183 pbMsrBitmap += 0x400;
1184 }
1185 else
1186 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1187
1188 Assert(iBit <= 0x1fff);
1189 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1190 ASMBitSet(pbMsrBitmap, iBit);
1191 else
1192 ASMBitClear(pbMsrBitmap, iBit);
1193
1194 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1195 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1196 else
1197 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1198}
1199
1200
1201#ifdef VBOX_STRICT
1202/**
1203 * Gets the permission bits for the specified MSR in the MSR bitmap.
1204 *
1205 * @returns VBox status code.
1206 * @retval VINF_SUCCESS if the specified MSR is found.
1207 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1208 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1209 *
1210 * @param pVCpu The cross context virtual CPU structure.
1211 * @param uMsr The MSR.
1212 * @param penmRead Where to store the read permissions.
1213 * @param penmWrite Where to store the write permissions.
1214 */
1215static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1216{
1217 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1218 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1219 int32_t iBit;
1220 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1221
1222 /* See hmR0VmxSetMsrPermission() for the layout. */
1223 if (uMsr <= 0x00001FFF)
1224 iBit = uMsr;
1225 else if ( uMsr >= 0xC0000000
1226 && uMsr <= 0xC0001FFF)
1227 {
1228 iBit = (uMsr - 0xC0000000);
1229 pbMsrBitmap += 0x400;
1230 }
1231 else
1232 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1233
1234 Assert(iBit <= 0x1fff);
1235 if (ASMBitTest(pbMsrBitmap, iBit))
1236 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1237 else
1238 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1239
1240 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1241 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1242 else
1243 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1244 return VINF_SUCCESS;
1245}
1246#endif /* VBOX_STRICT */
1247
1248
1249/**
1250 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1251 * area.
1252 *
1253 * @returns VBox status code.
1254 * @param pVCpu The cross context virtual CPU structure.
1255 * @param cMsrs The number of MSRs.
1256 */
1257static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1258{
1259 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1260 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1261 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1262 {
1263 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1264 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1265 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1266 }
1267
1268 /* Update number of guest MSRs to load/store across the world-switch. */
1269 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1270 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1271
1272 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1273 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1274 AssertRCReturn(rc, rc);
1275
1276 /* Update the VCPU's copy of the MSR count. */
1277 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1278
1279 return VINF_SUCCESS;
1280}
1281
1282
1283/**
1284 * Adds a new (or updates the value of an existing) guest/host MSR
1285 * pair to be swapped during the world-switch as part of the
1286 * auto-load/store MSR area in the VMCS.
1287 *
1288 * @returns VBox status code.
1289 * @param pVCpu The cross context virtual CPU structure.
1290 * @param uMsr The MSR.
1291 * @param uGuestMsrValue Value of the guest MSR.
1292 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1293 * necessary.
1294 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1295 * its value was updated. Optional, can be NULL.
1296 */
1297static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1298 bool *pfAddedAndUpdated)
1299{
1300 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1301 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1302 uint32_t i;
1303 for (i = 0; i < cMsrs; i++)
1304 {
1305 if (pGuestMsr->u32Msr == uMsr)
1306 break;
1307 pGuestMsr++;
1308 }
1309
1310 bool fAdded = false;
1311 if (i == cMsrs)
1312 {
1313 ++cMsrs;
1314 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1315 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1316
1317 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1318 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1319 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1320
1321 fAdded = true;
1322 }
1323
1324 /* Update the MSR values in the auto-load/store MSR area. */
1325 pGuestMsr->u32Msr = uMsr;
1326 pGuestMsr->u64Value = uGuestMsrValue;
1327
1328 /* Create/update the MSR slot in the host MSR area. */
1329 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1330 pHostMsr += i;
1331 pHostMsr->u32Msr = uMsr;
1332
1333 /*
1334 * Update the host MSR only when requested by the caller AND when we're
1335 * adding it to the auto-load/store area. Otherwise, it would have been
1336 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1337 */
1338 bool fUpdatedMsrValue = false;
1339 if ( fAdded
1340 && fUpdateHostMsr)
1341 {
1342 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1343 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1344 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1345 fUpdatedMsrValue = true;
1346 }
1347
1348 if (pfAddedAndUpdated)
1349 *pfAddedAndUpdated = fUpdatedMsrValue;
1350 return VINF_SUCCESS;
1351}
1352
1353
1354/**
1355 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1356 * auto-load/store MSR area in the VMCS.
1357 *
1358 * @returns VBox status code.
1359 * @param pVCpu The cross context virtual CPU structure.
1360 * @param uMsr The MSR.
1361 */
1362static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1363{
1364 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1365 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1366 for (uint32_t i = 0; i < cMsrs; i++)
1367 {
1368 /* Find the MSR. */
1369 if (pGuestMsr->u32Msr == uMsr)
1370 {
1371 /* If it's the last MSR, simply reduce the count. */
1372 if (i == cMsrs - 1)
1373 {
1374 --cMsrs;
1375 break;
1376 }
1377
1378 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1379 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1380 pLastGuestMsr += cMsrs - 1;
1381 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1382 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1383
1384 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1385 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1386 pLastHostMsr += cMsrs - 1;
1387 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1388 pHostMsr->u64Value = pLastHostMsr->u64Value;
1389 --cMsrs;
1390 break;
1391 }
1392 pGuestMsr++;
1393 }
1394
1395 /* Update the VMCS if the count changed (meaning the MSR was found). */
1396 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1397 {
1398 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1399 AssertRCReturn(rc, rc);
1400
1401 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1402 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1403 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1404
1405 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1406 return VINF_SUCCESS;
1407 }
1408
1409 return VERR_NOT_FOUND;
1410}
1411
1412
1413/**
1414 * Checks if the specified guest MSR is part of the auto-load/store area in
1415 * the VMCS.
1416 *
1417 * @returns true if found, false otherwise.
1418 * @param pVCpu The cross context virtual CPU structure.
1419 * @param uMsr The MSR to find.
1420 */
1421static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1422{
1423 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1424 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1425
1426 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1427 {
1428 if (pGuestMsr->u32Msr == uMsr)
1429 return true;
1430 }
1431 return false;
1432}
1433
1434
1435/**
1436 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1437 *
1438 * @param pVCpu The cross context virtual CPU structure.
1439 *
1440 * @remarks No-long-jump zone!!!
1441 */
1442static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1443{
1444 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1445 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1446 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1447 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1448
1449 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1450 {
1451 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1452
1453 /*
1454 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1455 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1456 */
1457 if (pHostMsr->u32Msr == MSR_K6_EFER)
1458 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1459 else
1460 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1461 }
1462
1463 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1464}
1465
1466
1467/**
1468 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1469 * perform lazy restoration of the host MSRs while leaving VT-x.
1470 *
1471 * @param pVCpu The cross context virtual CPU structure.
1472 *
1473 * @remarks No-long-jump zone!!!
1474 */
1475static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1476{
1477 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1478
1479 /*
1480 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1481 */
1482 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1483 {
1484 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1485#if HC_ARCH_BITS == 64
1486 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1487 {
1488 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1489 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1490 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1491 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1492 }
1493#endif
1494 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1495 }
1496}
1497
1498
1499/**
1500 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1501 * lazily while leaving VT-x.
1502 *
1503 * @returns true if it does, false otherwise.
1504 * @param pVCpu The cross context virtual CPU structure.
1505 * @param uMsr The MSR to check.
1506 */
1507static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1508{
1509 NOREF(pVCpu);
1510#if HC_ARCH_BITS == 64
1511 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1512 {
1513 switch (uMsr)
1514 {
1515 case MSR_K8_LSTAR:
1516 case MSR_K6_STAR:
1517 case MSR_K8_SF_MASK:
1518 case MSR_K8_KERNEL_GS_BASE:
1519 return true;
1520 }
1521 }
1522#else
1523 RT_NOREF(pVCpu, uMsr);
1524#endif
1525 return false;
1526}
1527
1528
1529/**
1530 * Loads a set of guests MSRs to allow read/passthru to the guest.
1531 *
1532 * The name of this function is slightly confusing. This function does NOT
1533 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1534 * common prefix for functions dealing with "lazy restoration" of the shared
1535 * MSRs.
1536 *
1537 * @param pVCpu The cross context virtual CPU structure.
1538 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1539 * out-of-sync. Make sure to update the required fields
1540 * before using them.
1541 *
1542 * @remarks No-long-jump zone!!!
1543 */
1544static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1545{
1546 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1547 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1548
1549 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1550#if HC_ARCH_BITS == 64
1551 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1552 {
1553 /*
1554 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1555 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1556 * we can skip a few MSR writes.
1557 *
1558 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1559 * guest MSR values in the guest-CPU context might be different to what's currently
1560 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1561 * CPU, see @bugref{8728}.
1562 */
1563 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1564 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1565 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1566 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1567 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1568 {
1569#ifdef VBOX_STRICT
1570 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1571 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1572 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1573 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1574#endif
1575 }
1576 else
1577 {
1578 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1579 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1580 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1581 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1582 }
1583 }
1584#else
1585 RT_NOREF(pMixedCtx);
1586#endif
1587 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1588}
1589
1590
1591/**
1592 * Performs lazy restoration of the set of host MSRs if they were previously
1593 * loaded with guest MSR values.
1594 *
1595 * @param pVCpu The cross context virtual CPU structure.
1596 *
1597 * @remarks No-long-jump zone!!!
1598 * @remarks The guest MSRs should have been saved back into the guest-CPU
1599 * context by hmR0VmxImportGuestState()!!!
1600 */
1601static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1602{
1603 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1604 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1605
1606 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1607 {
1608 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1609#if HC_ARCH_BITS == 64
1610 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1611 {
1612 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1613 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1614 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1615 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1616 }
1617#endif
1618 }
1619 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1620}
1621
1622
1623/**
1624 * Verifies that our cached values of the VMCS controls are all
1625 * consistent with what's actually present in the VMCS.
1626 *
1627 * @returns VBox status code.
1628 * @param pVCpu The cross context virtual CPU structure.
1629 */
1630static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1631{
1632 uint32_t u32Val;
1633 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1634 AssertRCReturn(rc, rc);
1635 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1636 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1637
1638 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1639 AssertRCReturn(rc, rc);
1640 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1641 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1642
1643 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1644 AssertRCReturn(rc, rc);
1645 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1646 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1647
1648 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1649 AssertRCReturn(rc, rc);
1650 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1651 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1652
1653 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1654 {
1655 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1656 AssertRCReturn(rc, rc);
1657 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1658 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1659 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1660 }
1661
1662 return VINF_SUCCESS;
1663}
1664
1665
1666#ifdef VBOX_STRICT
1667/**
1668 * Verifies that our cached host EFER value has not changed
1669 * since we cached it.
1670 *
1671 * @param pVCpu The cross context virtual CPU structure.
1672 */
1673static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1674{
1675 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1676
1677 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1678 {
1679 uint64_t u64Val;
1680 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1681 AssertRC(rc);
1682
1683 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1684 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1685 }
1686}
1687
1688
1689/**
1690 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1691 * VMCS are correct.
1692 *
1693 * @param pVCpu The cross context virtual CPU structure.
1694 */
1695static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1696{
1697 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1698
1699 /* Verify MSR counts in the VMCS are what we think it should be. */
1700 uint32_t cMsrs;
1701 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1702 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1703
1704 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1705 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1706
1707 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1708 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1709
1710 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1711 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1712 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1713 {
1714 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1715 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1716 pGuestMsr->u32Msr, cMsrs));
1717
1718 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1719 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1720 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1721
1722 /* Verify that the permissions are as expected in the MSR bitmap. */
1723 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1724 {
1725 VMXMSREXITREAD enmRead;
1726 VMXMSREXITWRITE enmWrite;
1727 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1728 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1729 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1730 {
1731 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1732 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1733 }
1734 else
1735 {
1736 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1737 pGuestMsr->u32Msr, cMsrs));
1738 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1739 pGuestMsr->u32Msr, cMsrs));
1740 }
1741 }
1742 }
1743}
1744#endif /* VBOX_STRICT */
1745
1746
1747/**
1748 * Flushes the TLB using EPT.
1749 *
1750 * @returns VBox status code.
1751 * @param pVCpu The cross context virtual CPU structure of the calling
1752 * EMT. Can be NULL depending on @a enmFlush.
1753 * @param enmFlush Type of flush.
1754 *
1755 * @remarks Caller is responsible for making sure this function is called only
1756 * when NestedPaging is supported and providing @a enmFlush that is
1757 * supported by the CPU.
1758 * @remarks Can be called with interrupts disabled.
1759 */
1760static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1761{
1762 uint64_t au64Descriptor[2];
1763 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1764 au64Descriptor[0] = 0;
1765 else
1766 {
1767 Assert(pVCpu);
1768 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1769 }
1770 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1771
1772 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1773 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1774 rc));
1775 if ( RT_SUCCESS(rc)
1776 && pVCpu)
1777 {
1778 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1779 }
1780}
1781
1782
1783/**
1784 * Flushes the TLB using VPID.
1785 *
1786 * @returns VBox status code.
1787 * @param pVCpu The cross context virtual CPU structure of the calling
1788 * EMT. Can be NULL depending on @a enmFlush.
1789 * @param enmFlush Type of flush.
1790 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1791 * on @a enmFlush).
1792 *
1793 * @remarks Can be called with interrupts disabled.
1794 */
1795static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1796{
1797 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1798
1799 uint64_t au64Descriptor[2];
1800 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1801 {
1802 au64Descriptor[0] = 0;
1803 au64Descriptor[1] = 0;
1804 }
1805 else
1806 {
1807 AssertPtr(pVCpu);
1808 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1809 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1810 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1811 au64Descriptor[1] = GCPtr;
1812 }
1813
1814 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1815 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1816 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1817 if ( RT_SUCCESS(rc)
1818 && pVCpu)
1819 {
1820 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1821 }
1822 NOREF(rc);
1823}
1824
1825
1826/**
1827 * Invalidates a guest page by guest virtual address. Only relevant for
1828 * EPT/VPID, otherwise there is nothing really to invalidate.
1829 *
1830 * @returns VBox status code.
1831 * @param pVCpu The cross context virtual CPU structure.
1832 * @param GCVirt Guest virtual address of the page to invalidate.
1833 */
1834VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1835{
1836 AssertPtr(pVCpu);
1837 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1838
1839 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1840 if (!fFlushPending)
1841 {
1842 /*
1843 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1844 * the EPT case. See @bugref{6043} and @bugref{6177}.
1845 *
1846 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1847 * as this function maybe called in a loop with individual addresses.
1848 */
1849 PVM pVM = pVCpu->CTX_SUFF(pVM);
1850 if (pVM->hm.s.vmx.fVpid)
1851 {
1852 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1853
1854#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1855 /*
1856 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1857 * where executing INVVPID outside 64-bit mode does not flush translations of
1858 * 64-bit linear addresses, see @bugref{6208#c72}.
1859 */
1860 if (RT_HI_U32(GCVirt))
1861 fVpidFlush = false;
1862#endif
1863
1864 if (fVpidFlush)
1865 {
1866 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1867 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1868 }
1869 else
1870 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1871 }
1872 else if (pVM->hm.s.fNestedPaging)
1873 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1874 }
1875
1876 return VINF_SUCCESS;
1877}
1878
1879
1880/**
1881 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1882 * case where neither EPT nor VPID is supported by the CPU.
1883 *
1884 * @param pVCpu The cross context virtual CPU structure.
1885 * @param pCpu Pointer to the global HM struct.
1886 *
1887 * @remarks Called with interrupts disabled.
1888 */
1889static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1890{
1891 AssertPtr(pVCpu);
1892 AssertPtr(pCpu);
1893
1894 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1895
1896 Assert(pCpu->idCpu != NIL_RTCPUID);
1897 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1898 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1899 pVCpu->hm.s.fForceTLBFlush = false;
1900 return;
1901}
1902
1903
1904/**
1905 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1906 *
1907 * @param pVCpu The cross context virtual CPU structure.
1908 * @param pCpu Pointer to the global HM CPU struct.
1909 *
1910 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1911 * nomenclature. The reason is, to avoid confusion in compare statements
1912 * since the host-CPU copies are named "ASID".
1913 *
1914 * @remarks Called with interrupts disabled.
1915 */
1916static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1917{
1918#ifdef VBOX_WITH_STATISTICS
1919 bool fTlbFlushed = false;
1920# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1921# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1922 if (!fTlbFlushed) \
1923 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1924 } while (0)
1925#else
1926# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1927# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1928#endif
1929
1930 AssertPtr(pCpu);
1931 AssertPtr(pVCpu);
1932 Assert(pCpu->idCpu != NIL_RTCPUID);
1933
1934 PVM pVM = pVCpu->CTX_SUFF(pVM);
1935 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1936 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1937 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1938
1939 /*
1940 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1941 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1942 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1943 * cannot reuse the current ASID anymore.
1944 */
1945 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1946 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1947 {
1948 ++pCpu->uCurrentAsid;
1949 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1950 {
1951 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1952 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1953 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1954 }
1955
1956 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1957 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1958 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1959
1960 /*
1961 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1962 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1963 */
1964 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1965 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1966 HMVMX_SET_TAGGED_TLB_FLUSHED();
1967 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1968 }
1969
1970 /* Check for explicit TLB flushes. */
1971 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1972 {
1973 /*
1974 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1975 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1976 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1977 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1978 * mappings, see @bugref{6568}.
1979 *
1980 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1981 */
1982 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1983 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1984 HMVMX_SET_TAGGED_TLB_FLUSHED();
1985 }
1986
1987 pVCpu->hm.s.fForceTLBFlush = false;
1988 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1989
1990 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1991 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1992 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1993 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1994 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1995 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1996 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1997 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1998 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1999
2000 /* Update VMCS with the VPID. */
2001 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2002 AssertRC(rc);
2003
2004#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2005}
2006
2007
2008/**
2009 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2010 *
2011 * @returns VBox status code.
2012 * @param pVCpu The cross context virtual CPU structure.
2013 * @param pCpu Pointer to the global HM CPU struct.
2014 *
2015 * @remarks Called with interrupts disabled.
2016 */
2017static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2018{
2019 AssertPtr(pVCpu);
2020 AssertPtr(pCpu);
2021 Assert(pCpu->idCpu != NIL_RTCPUID);
2022 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2023 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2024
2025 /*
2026 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2027 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2028 */
2029 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2030 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2031 {
2032 pVCpu->hm.s.fForceTLBFlush = true;
2033 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2034 }
2035
2036 /* Check for explicit TLB flushes. */
2037 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2038 {
2039 pVCpu->hm.s.fForceTLBFlush = true;
2040 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2041 }
2042
2043 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2044 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2045
2046 if (pVCpu->hm.s.fForceTLBFlush)
2047 {
2048 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2049 pVCpu->hm.s.fForceTLBFlush = false;
2050 }
2051}
2052
2053
2054/**
2055 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2056 *
2057 * @returns VBox status code.
2058 * @param pVCpu The cross context virtual CPU structure.
2059 * @param pCpu Pointer to the global HM CPU struct.
2060 *
2061 * @remarks Called with interrupts disabled.
2062 */
2063static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2064{
2065 AssertPtr(pVCpu);
2066 AssertPtr(pCpu);
2067 Assert(pCpu->idCpu != NIL_RTCPUID);
2068 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2069 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2070
2071 /*
2072 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2073 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2074 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2075 * cannot reuse the current ASID anymore.
2076 */
2077 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2078 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2079 {
2080 pVCpu->hm.s.fForceTLBFlush = true;
2081 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2082 }
2083
2084 /* Check for explicit TLB flushes. */
2085 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2086 {
2087 /*
2088 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2089 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2090 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2091 * include fExplicitFlush's too) - an obscure corner case.
2092 */
2093 pVCpu->hm.s.fForceTLBFlush = true;
2094 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2095 }
2096
2097 PVM pVM = pVCpu->CTX_SUFF(pVM);
2098 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2099 if (pVCpu->hm.s.fForceTLBFlush)
2100 {
2101 ++pCpu->uCurrentAsid;
2102 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2103 {
2104 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2105 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2106 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2107 }
2108
2109 pVCpu->hm.s.fForceTLBFlush = false;
2110 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2111 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2112 if (pCpu->fFlushAsidBeforeUse)
2113 {
2114 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2115 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2116 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2117 {
2118 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2119 pCpu->fFlushAsidBeforeUse = false;
2120 }
2121 else
2122 {
2123 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2124 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2125 }
2126 }
2127 }
2128
2129 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2130 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2131 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2132 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2133 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2134 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2135 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2136
2137 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2138 AssertRC(rc);
2139}
2140
2141
2142/**
2143 * Flushes the guest TLB entry based on CPU capabilities.
2144 *
2145 * @param pVCpu The cross context virtual CPU structure.
2146 * @param pCpu Pointer to the global HM CPU struct.
2147 */
2148DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2149{
2150#ifdef HMVMX_ALWAYS_FLUSH_TLB
2151 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2152#endif
2153 PVM pVM = pVCpu->CTX_SUFF(pVM);
2154 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2155 {
2156 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2158 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2159 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2160 default:
2161 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2162 break;
2163 }
2164 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2165}
2166
2167
2168/**
2169 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2170 * TLB entries from the host TLB before VM-entry.
2171 *
2172 * @returns VBox status code.
2173 * @param pVM The cross context VM structure.
2174 */
2175static int hmR0VmxSetupTaggedTlb(PVM pVM)
2176{
2177 /*
2178 * Determine optimal flush type for Nested Paging.
2179 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2180 * guest execution (see hmR3InitFinalizeR0()).
2181 */
2182 if (pVM->hm.s.fNestedPaging)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2185 {
2186 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2187 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2188 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2189 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2190 else
2191 {
2192 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2193 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2194 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2195 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2196 }
2197
2198 /* Make sure the write-back cacheable memory type for EPT is supported. */
2199 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2200 {
2201 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2202 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2203 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2204 }
2205
2206 /* EPT requires a page-walk length of 4. */
2207 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2208 {
2209 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2210 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2211 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2212 }
2213 }
2214 else
2215 {
2216 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2217 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2218 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2219 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2220 }
2221 }
2222
2223 /*
2224 * Determine optimal flush type for VPID.
2225 */
2226 if (pVM->hm.s.vmx.fVpid)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2231 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2232 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2233 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2234 else
2235 {
2236 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2238 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2240 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2241 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2242 pVM->hm.s.vmx.fVpid = false;
2243 }
2244 }
2245 else
2246 {
2247 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2248 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2249 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2250 pVM->hm.s.vmx.fVpid = false;
2251 }
2252 }
2253
2254 /*
2255 * Setup the handler for flushing tagged-TLBs.
2256 */
2257 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2258 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2259 else if (pVM->hm.s.fNestedPaging)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2261 else if (pVM->hm.s.vmx.fVpid)
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2263 else
2264 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2265 return VINF_SUCCESS;
2266}
2267
2268
2269/**
2270 * Sets up pin-based VM-execution controls in the VMCS.
2271 *
2272 * @returns VBox status code.
2273 * @param pVCpu The cross context virtual CPU structure.
2274 *
2275 * @remarks We don't really care about optimizing vmwrites here as it's done only
2276 * once per VM and hence we don't care about VMCS-field cache comparisons.
2277 */
2278static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2279{
2280 PVM pVM = pVCpu->CTX_SUFF(pVM);
2281 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2282 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2283
2284 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2285 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2286
2287 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2288 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2289
2290 /* Enable the VMX preemption timer. */
2291 if (pVM->hm.s.vmx.fUsePreemptTimer)
2292 {
2293 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2294 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2295 }
2296
2297#if 0
2298 /* Enable posted-interrupt processing. */
2299 if (pVM->hm.s.fPostedIntrs)
2300 {
2301 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2302 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2303 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2304 }
2305#endif
2306
2307 if ((fVal & fZap) != fVal)
2308 {
2309 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2310 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2311 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2312 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2313 }
2314
2315 /* Commit it to the VMCS and update our cache. */
2316 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2317 AssertRCReturn(rc, rc);
2318 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2319
2320 return VINF_SUCCESS;
2321}
2322
2323
2324/**
2325 * Sets up secondary processor-based VM-execution controls in the VMCS.
2326 *
2327 * @returns VBox status code.
2328 * @param pVCpu The cross context virtual CPU structure.
2329 *
2330 * @remarks We don't really care about optimizing vmwrites here as it's done only
2331 * once per VM and hence we don't care about VMCS-field cache comparisons.
2332 */
2333static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2334{
2335 PVM pVM = pVCpu->CTX_SUFF(pVM);
2336 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2337 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2338
2339 /* WBINVD causes a VM-exit. */
2340 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2341 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2342
2343 /* Enable EPT (aka nested-paging). */
2344 if (pVM->hm.s.fNestedPaging)
2345 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2346
2347 /*
2348 * Enable the INVPCID instruction if supported by the hardware and we expose
2349 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2350 */
2351 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2352 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2353 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2354
2355 /* Enable VPID. */
2356 if (pVM->hm.s.vmx.fVpid)
2357 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2358
2359 /* Enable Unrestricted guest execution. */
2360 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2361 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2362
2363#if 0
2364 if (pVM->hm.s.fVirtApicRegs)
2365 {
2366 /* Enable APIC-register virtualization. */
2367 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2368 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2369
2370 /* Enable virtual-interrupt delivery. */
2371 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2372 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2373 }
2374#endif
2375
2376 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2377 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2378 * done dynamically. */
2379 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2380 {
2381 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2382 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2383 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2384 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2385 AssertRCReturn(rc, rc);
2386 }
2387
2388 /* Enable RDTSCP. */
2389 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2390 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2391
2392 /* Enable Pause-Loop exiting. */
2393 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2394 && pVM->hm.s.vmx.cPleGapTicks
2395 && pVM->hm.s.vmx.cPleWindowTicks)
2396 {
2397 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2398
2399 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2400 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2401 AssertRCReturn(rc, rc);
2402 }
2403
2404 if ((fVal & fZap) != fVal)
2405 {
2406 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2407 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2408 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2409 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2410 }
2411
2412 /* Commit it to the VMCS and update our cache. */
2413 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2414 AssertRCReturn(rc, rc);
2415 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2416
2417 return VINF_SUCCESS;
2418}
2419
2420
2421/**
2422 * Sets up processor-based VM-execution controls in the VMCS.
2423 *
2424 * @returns VBox status code.
2425 * @param pVCpu The cross context virtual CPU structure.
2426 *
2427 * @remarks We don't really care about optimizing vmwrites here as it's done only
2428 * once per VM and hence we don't care about VMCS-field cache comparisons.
2429 */
2430static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2431{
2432 PVM pVM = pVCpu->CTX_SUFF(pVM);
2433 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2434 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2435
2436 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2437 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2438 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2441 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2442 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2443
2444 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2445 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2446 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2447 {
2448 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2449 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2450 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2451 }
2452
2453 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2454 if (!pVM->hm.s.fNestedPaging)
2455 {
2456 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2457 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2458 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2459 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2460 }
2461
2462 /* Use TPR shadowing if supported by the CPU. */
2463 if ( PDMHasApic(pVM)
2464 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2465 {
2466 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2467 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2468 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2469 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2470 AssertRCReturn(rc, rc);
2471
2472 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2473 /* CR8 writes cause a VM-exit based on TPR threshold. */
2474 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2475 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2476 }
2477 else
2478 {
2479 /*
2480 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2481 * Set this control only for 64-bit guests.
2482 */
2483 if (pVM->hm.s.fAllow64BitGuests)
2484 {
2485 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2486 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2487 }
2488 }
2489
2490 /* Use MSR-bitmaps if supported by the CPU. */
2491 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2492 {
2493 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2494
2495 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2497 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2498 AssertRCReturn(rc, rc);
2499
2500 /*
2501 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2502 * automatically using dedicated fields in the VMCS.
2503 */
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2509#if HC_ARCH_BITS == 64
2510 /*
2511 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2512 */
2513 if (pVM->hm.s.fAllow64BitGuests)
2514 {
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 }
2520#endif
2521 /*
2522 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2523 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2524 */
2525 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2526 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2527
2528 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2529 }
2530
2531 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2532 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2533 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2534
2535 if ((fVal & fZap) != fVal)
2536 {
2537 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2538 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2539 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2540 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2541 }
2542
2543 /* Commit it to the VMCS and update our cache. */
2544 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2545 AssertRCReturn(rc, rc);
2546 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2547
2548 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2549 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2550 return hmR0VmxSetupProcCtls2(pVCpu);
2551
2552 /* Sanity check, should not really happen. */
2553 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2554 {
2555 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2556 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2557 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2558 }
2559
2560 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2561 return VINF_SUCCESS;
2562}
2563
2564
2565/**
2566 * Sets up miscellaneous (everything other than Pin & Processor-based
2567 * VM-execution) control fields in the VMCS.
2568 *
2569 * @returns VBox status code.
2570 * @param pVCpu The cross context virtual CPU structure.
2571 */
2572static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2573{
2574 AssertPtr(pVCpu);
2575
2576 int rc = VERR_GENERAL_FAILURE;
2577
2578 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2579#if 0
2580 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2581 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2582 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2583
2584 /*
2585 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2586 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2587 * We thus use the exception bitmap to control it rather than use both.
2588 */
2589 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2590 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2591
2592 /* All IO & IOIO instructions cause VM-exits. */
2593 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2594 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2595
2596 /* Initialize the MSR-bitmap area. */
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2598 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2600 AssertRCReturn(rc, rc);
2601#endif
2602
2603 /* Setup MSR auto-load/store area. */
2604 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2606 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2608 AssertRCReturn(rc, rc);
2609
2610 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2612 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2613 AssertRCReturn(rc, rc);
2614
2615 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2616 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2617 AssertRCReturn(rc, rc);
2618
2619 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2620#if 0
2621 /* Setup debug controls */
2622 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2623 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2624 AssertRCReturn(rc, rc);
2625#endif
2626
2627 return rc;
2628}
2629
2630
2631/**
2632 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2633 *
2634 * We shall setup those exception intercepts that don't change during the
2635 * lifetime of the VM here. The rest are done dynamically while loading the
2636 * guest state.
2637 *
2638 * @returns VBox status code.
2639 * @param pVCpu The cross context virtual CPU structure.
2640 */
2641static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2642{
2643 AssertPtr(pVCpu);
2644
2645 uint32_t u32XcptBitmap;
2646
2647 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2648 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2649
2650 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2651 and writes, and because recursive #DBs can cause the CPU hang, we must always
2652 intercept #DB. */
2653 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2654
2655 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2656 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2657 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2658
2659 /* Commit it to the VMCS. */
2660 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2661 AssertRCReturn(rc, rc);
2662
2663 /* Update our cache of the exception bitmap. */
2664 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2665 return VINF_SUCCESS;
2666}
2667
2668
2669/**
2670 * Does per-VM VT-x initialization.
2671 *
2672 * @returns VBox status code.
2673 * @param pVM The cross context VM structure.
2674 */
2675VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2676{
2677 LogFlowFunc(("pVM=%p\n", pVM));
2678
2679 int rc = hmR0VmxStructsAlloc(pVM);
2680 if (RT_FAILURE(rc))
2681 {
2682 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2683 return rc;
2684 }
2685
2686 return VINF_SUCCESS;
2687}
2688
2689
2690/**
2691 * Does per-VM VT-x termination.
2692 *
2693 * @returns VBox status code.
2694 * @param pVM The cross context VM structure.
2695 */
2696VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2697{
2698 LogFlowFunc(("pVM=%p\n", pVM));
2699
2700#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2701 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2702 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2703#endif
2704 hmR0VmxStructsFree(pVM);
2705 return VINF_SUCCESS;
2706}
2707
2708
2709/**
2710 * Sets up the VM for execution under VT-x.
2711 * This function is only called once per-VM during initialization.
2712 *
2713 * @returns VBox status code.
2714 * @param pVM The cross context VM structure.
2715 */
2716VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2717{
2718 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2719 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2720
2721 LogFlowFunc(("pVM=%p\n", pVM));
2722
2723 /*
2724 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2725 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2726 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2727 */
2728 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2729 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2730 || !pVM->hm.s.vmx.pRealModeTSS))
2731 {
2732 LogRelFunc(("Invalid real-on-v86 state.\n"));
2733 return VERR_INTERNAL_ERROR;
2734 }
2735
2736 /* Initialize these always, see hmR3InitFinalizeR0().*/
2737 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2738 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2739
2740 /* Setup the tagged-TLB flush handlers. */
2741 int rc = hmR0VmxSetupTaggedTlb(pVM);
2742 if (RT_FAILURE(rc))
2743 {
2744 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2745 return rc;
2746 }
2747
2748 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2749 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2750#if HC_ARCH_BITS == 64
2751 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2752 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2753 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2754 {
2755 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2756 }
2757#endif
2758
2759 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2760 RTCCUINTREG uHostCR4 = ASMGetCR4();
2761 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2762 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2763
2764 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2765 {
2766 PVMCPU pVCpu = &pVM->aCpus[i];
2767 AssertPtr(pVCpu);
2768 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2769
2770 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2771 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2772
2773 /* Set revision dword at the beginning of the VMCS structure. */
2774 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2775
2776 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2777 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2778 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2779 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2780
2781 /* Load this VMCS as the current VMCS. */
2782 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2783 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2784 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2785
2786 rc = hmR0VmxSetupPinCtls(pVCpu);
2787 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2788 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2789
2790 rc = hmR0VmxSetupProcCtls(pVCpu);
2791 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2792 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2793
2794 rc = hmR0VmxSetupMiscCtls(pVCpu);
2795 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2796 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2797
2798 rc = hmR0VmxInitXcptBitmap(pVCpu);
2799 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2800 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2801
2802#if HC_ARCH_BITS == 32
2803 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2804 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2805 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2806#endif
2807
2808 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2809 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2810 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2811 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2812
2813 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2814
2815 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2816 }
2817
2818 return VINF_SUCCESS;
2819}
2820
2821
2822/**
2823 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2824 * the VMCS.
2825 *
2826 * @returns VBox status code.
2827 */
2828static int hmR0VmxExportHostControlRegs(void)
2829{
2830 RTCCUINTREG uReg = ASMGetCR0();
2831 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2832 AssertRCReturn(rc, rc);
2833
2834 uReg = ASMGetCR3();
2835 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2836 AssertRCReturn(rc, rc);
2837
2838 uReg = ASMGetCR4();
2839 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2840 AssertRCReturn(rc, rc);
2841 return rc;
2842}
2843
2844
2845/**
2846 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2847 * the host-state area in the VMCS.
2848 *
2849 * @returns VBox status code.
2850 * @param pVCpu The cross context virtual CPU structure.
2851 */
2852static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2853{
2854#if HC_ARCH_BITS == 64
2855/**
2856 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2857 * requirements. See hmR0VmxExportHostSegmentRegs().
2858 */
2859# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2860 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2861 { \
2862 bool fValidSelector = true; \
2863 if ((selValue) & X86_SEL_LDT) \
2864 { \
2865 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2866 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2867 } \
2868 if (fValidSelector) \
2869 { \
2870 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2871 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2872 } \
2873 (selValue) = 0; \
2874 }
2875
2876 /*
2877 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2878 * should -not- save the messed up state without restoring the original host-state,
2879 * see @bugref{7240}.
2880 *
2881 * This apparently can happen (most likely the FPU changes), deal with it rather than
2882 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2883 */
2884 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2885 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2886 {
2887 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2888 pVCpu->idCpu));
2889 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2890 }
2891 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2892#else
2893 RT_NOREF(pVCpu);
2894#endif
2895
2896 /*
2897 * Host DS, ES, FS and GS segment registers.
2898 */
2899#if HC_ARCH_BITS == 64
2900 RTSEL uSelDS = ASMGetDS();
2901 RTSEL uSelES = ASMGetES();
2902 RTSEL uSelFS = ASMGetFS();
2903 RTSEL uSelGS = ASMGetGS();
2904#else
2905 RTSEL uSelDS = 0;
2906 RTSEL uSelES = 0;
2907 RTSEL uSelFS = 0;
2908 RTSEL uSelGS = 0;
2909#endif
2910
2911 /*
2912 * Host CS and SS segment registers.
2913 */
2914 RTSEL uSelCS = ASMGetCS();
2915 RTSEL uSelSS = ASMGetSS();
2916
2917 /*
2918 * Host TR segment register.
2919 */
2920 RTSEL uSelTR = ASMGetTR();
2921
2922#if HC_ARCH_BITS == 64
2923 /*
2924 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2925 * gain VM-entry and restore them before we get preempted.
2926 *
2927 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2928 */
2929 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2930 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2931 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2932 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2933# undef VMXLOCAL_ADJUST_HOST_SEG
2934#endif
2935
2936 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2937 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2938 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2939 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2940 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2941 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2942 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2943 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2944 Assert(uSelCS);
2945 Assert(uSelTR);
2946
2947 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2948#if 0
2949 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2950 Assert(uSelSS != 0);
2951#endif
2952
2953 /* Write these host selector fields into the host-state area in the VMCS. */
2954 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2956#if HC_ARCH_BITS == 64
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2959 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2960 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2961#else
2962 NOREF(uSelDS);
2963 NOREF(uSelES);
2964 NOREF(uSelFS);
2965 NOREF(uSelGS);
2966#endif
2967 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2968 AssertRCReturn(rc, rc);
2969
2970 /*
2971 * Host GDTR and IDTR.
2972 */
2973 RTGDTR Gdtr;
2974 RTIDTR Idtr;
2975 RT_ZERO(Gdtr);
2976 RT_ZERO(Idtr);
2977 ASMGetGDTR(&Gdtr);
2978 ASMGetIDTR(&Idtr);
2979 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2980 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2981 AssertRCReturn(rc, rc);
2982
2983#if HC_ARCH_BITS == 64
2984 /*
2985 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2986 * them to the maximum limit (0xffff) on every VM-exit.
2987 */
2988 if (Gdtr.cbGdt != 0xffff)
2989 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2990
2991 /*
2992 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2993 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2994 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2995 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2996 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2997 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2998 * at 0xffff on hosts where we are sure it won't cause trouble.
2999 */
3000# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3001 if (Idtr.cbIdt < 0x0fff)
3002# else
3003 if (Idtr.cbIdt != 0xffff)
3004# endif
3005 {
3006 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3007 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3008 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3009 }
3010#endif
3011
3012 /*
3013 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3014 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3015 * RPL should be too in most cases.
3016 */
3017 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3018 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3019
3020 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3021#if HC_ARCH_BITS == 64
3022 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3023
3024 /*
3025 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3026 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3027 * restoration if the host has something else. Task switching is not supported in 64-bit
3028 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3029 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3030 *
3031 * [1] See Intel spec. 3.5 "System Descriptor Types".
3032 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3033 */
3034 PVM pVM = pVCpu->CTX_SUFF(pVM);
3035 Assert(pDesc->System.u4Type == 11);
3036 if ( pDesc->System.u16LimitLow != 0x67
3037 || pDesc->System.u4LimitHigh)
3038 {
3039 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3040 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3041 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3042 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3043 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3044 }
3045
3046 /*
3047 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3048 */
3049 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3050 {
3051 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3052 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3053 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3054 {
3055 /* The GDT is read-only but the writable GDT is available. */
3056 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3057 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3058 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3059 AssertRCReturn(rc, rc);
3060 }
3061 }
3062#else
3063 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3064#endif
3065 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3066 AssertRCReturn(rc, rc);
3067
3068 /*
3069 * Host FS base and GS base.
3070 */
3071#if HC_ARCH_BITS == 64
3072 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3073 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3074 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3075 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3076 AssertRCReturn(rc, rc);
3077
3078 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3079 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3080 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3081 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3082 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3083#endif
3084 return VINF_SUCCESS;
3085}
3086
3087
3088/**
3089 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3090 * host-state area of the VMCS.
3091 *
3092 * Theses MSRs will be automatically restored on the host after every successful
3093 * VM-exit.
3094 *
3095 * @returns VBox status code.
3096 * @param pVCpu The cross context virtual CPU structure.
3097 *
3098 * @remarks No-long-jump zone!!!
3099 */
3100static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3101{
3102 AssertPtr(pVCpu);
3103 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3104
3105 /*
3106 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3107 * rather than swapping them on every VM-entry.
3108 */
3109 hmR0VmxLazySaveHostMsrs(pVCpu);
3110
3111 /*
3112 * Host Sysenter MSRs.
3113 */
3114 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3115#if HC_ARCH_BITS == 32
3116 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3117 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3118#else
3119 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3120 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3121#endif
3122 AssertRCReturn(rc, rc);
3123
3124 /*
3125 * Host EFER MSR.
3126 *
3127 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3128 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3129 */
3130 PVM pVM = pVCpu->CTX_SUFF(pVM);
3131 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3132 {
3133 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3134 AssertRCReturn(rc, rc);
3135 }
3136
3137 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3138
3139 return VINF_SUCCESS;
3140}
3141
3142
3143/**
3144 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3145 *
3146 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3147 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3148 * hmR0VMxExportGuestEntryCtls().
3149 *
3150 * @returns true if we need to load guest EFER, false otherwise.
3151 * @param pVCpu The cross context virtual CPU structure.
3152 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3153 * out-of-sync. Make sure to update the required fields
3154 * before using them.
3155 *
3156 * @remarks Requires EFER, CR4.
3157 * @remarks No-long-jump zone!!!
3158 */
3159static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3160{
3161#ifdef HMVMX_ALWAYS_SWAP_EFER
3162 return true;
3163#endif
3164
3165#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3166 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3167 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3168 return false;
3169#endif
3170
3171 PVM pVM = pVCpu->CTX_SUFF(pVM);
3172 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3173 uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3174
3175 /*
3176 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3177 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3178 */
3179 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3180 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3181 {
3182 return true;
3183 }
3184
3185 /*
3186 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3187 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3188 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3189 */
3190 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3191 && (pMixedCtx->cr0 & X86_CR0_PG)
3192 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3193 {
3194 /* Assert that host is PAE capable. */
3195 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3196 return true;
3197 }
3198
3199 return false;
3200}
3201
3202
3203/**
3204 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3205 *
3206 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3207 * see Intel spec. 24.8.1 "VM-entry controls".
3208 *
3209 * @returns VBox status code.
3210 * @param pVCpu The cross context virtual CPU structure.
3211 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3212 * out-of-sync. Make sure to update the required fields
3213 * before using them.
3214 *
3215 * @remarks Requires EFER.
3216 * @remarks No-long-jump zone!!!
3217 */
3218static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3219{
3220 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3221 {
3222 PVM pVM = pVCpu->CTX_SUFF(pVM);
3223 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3224 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3225
3226 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3227 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3228
3229 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3230 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3231 {
3232 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3233 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3234 }
3235 else
3236 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3237
3238 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3239 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3240 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3241 {
3242 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3243 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3244 }
3245
3246 /*
3247 * The following should -not- be set (since we're not in SMM mode):
3248 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3249 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3250 */
3251
3252 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3253 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3254
3255 if ((fVal & fZap) != fVal)
3256 {
3257 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3258 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3259 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3260 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3261 }
3262
3263 /* Commit it to the VMCS and update our cache. */
3264 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3265 {
3266 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3267 AssertRCReturn(rc, rc);
3268 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3269 }
3270
3271 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3272 }
3273 return VINF_SUCCESS;
3274}
3275
3276
3277/**
3278 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3279 *
3280 * @returns VBox status code.
3281 * @param pVCpu The cross context virtual CPU structure.
3282 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3283 * out-of-sync. Make sure to update the required fields
3284 * before using them.
3285 *
3286 * @remarks Requires EFER.
3287 */
3288static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3289{
3290 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3291 {
3292 PVM pVM = pVCpu->CTX_SUFF(pVM);
3293 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3294 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3295
3296 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3297 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3298
3299 /*
3300 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3301 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3302 * hmR0VmxExportHostMsrs().
3303 */
3304#if HC_ARCH_BITS == 64
3305 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3306 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3307#else
3308 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3309 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3310 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3311 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3312 {
3313 /* The switcher returns to long mode, EFER is managed by the switcher. */
3314 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3315 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3316 }
3317 else
3318 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3319#endif
3320
3321 /* If the newer VMCS fields for managing EFER exists, use it. */
3322 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3323 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3324 {
3325 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3326 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3327 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3328 }
3329
3330 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3331 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3332
3333 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3334 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3335 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3336
3337 /* Enable saving of the VMX preemption timer value on VM-exit. */
3338 if ( pVM->hm.s.vmx.fUsePreemptTimer
3339 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3340 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3341
3342 if ((fVal & fZap) != fVal)
3343 {
3344 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3345 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3346 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3347 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3348 }
3349
3350 /* Commit it to the VMCS and update our cache. */
3351 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3352 {
3353 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3354 AssertRCReturn(rc, rc);
3355 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3356 }
3357
3358 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3359 }
3360 return VINF_SUCCESS;
3361}
3362
3363
3364/**
3365 * Sets the TPR threshold in the VMCS.
3366 *
3367 * @returns VBox status code.
3368 * @param pVCpu The cross context virtual CPU structure.
3369 * @param u32TprThreshold The TPR threshold (task-priority class only).
3370 */
3371DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3372{
3373 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3374 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3375 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3376}
3377
3378
3379/**
3380 * Exports the guest APIC TPR state into the VMCS.
3381 *
3382 * @returns VBox status code.
3383 * @param pVCpu The cross context virtual CPU structure.
3384 *
3385 * @remarks No-long-jump zone!!!
3386 */
3387static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3388{
3389 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3390 {
3391 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_APIC_TPR);
3392
3393 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3394 && APICIsEnabled(pVCpu))
3395 {
3396 /*
3397 * Setup TPR shadowing.
3398 */
3399 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3400 {
3401 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3402
3403 bool fPendingIntr = false;
3404 uint8_t u8Tpr = 0;
3405 uint8_t u8PendingIntr = 0;
3406 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3407 AssertRCReturn(rc, rc);
3408
3409 /*
3410 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3411 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3412 * priority of the pending interrupt so we can deliver the interrupt. If there
3413 * are no interrupts pending, set threshold to 0 to not cause any
3414 * TPR-below-threshold VM-exits.
3415 */
3416 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3417 uint32_t u32TprThreshold = 0;
3418 if (fPendingIntr)
3419 {
3420 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3421 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3422 const uint8_t u8TprPriority = u8Tpr >> 4;
3423 if (u8PendingPriority <= u8TprPriority)
3424 u32TprThreshold = u8PendingPriority;
3425 }
3426
3427 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3428 AssertRCReturn(rc, rc);
3429 }
3430 }
3431 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3432 }
3433 return VINF_SUCCESS;
3434}
3435
3436
3437/**
3438 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3439 *
3440 * @returns Guest's interruptibility-state.
3441 * @param pVCpu The cross context virtual CPU structure.
3442 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3443 * out-of-sync. Make sure to update the required fields
3444 * before using them.
3445 *
3446 * @remarks No-long-jump zone!!!
3447 */
3448static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3449{
3450 /*
3451 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452 */
3453 uint32_t fIntrState = 0;
3454 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /* If inhibition is active, RIP & RFLAGS should've been accessed
3457 (i.e. read previously from the VMCS or from ring-3). */
3458#ifdef VBOX_STRICT
3459 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3460 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3461#endif
3462 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3463 {
3464 if (pMixedCtx->eflags.Bits.u1IF)
3465 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3466 else
3467 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3468 }
3469 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3470 {
3471 /*
3472 * We can clear the inhibit force flag as even if we go back to the recompiler
3473 * without executing guest code in VT-x, the flag's condition to be cleared is
3474 * met and thus the cleared state is correct.
3475 */
3476 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3477 }
3478 }
3479
3480 /*
3481 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3482 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3483 * setting this would block host-NMIs and IRET will not clear the blocking.
3484 *
3485 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3486 */
3487 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3488 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3489 {
3490 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3491 }
3492
3493 return fIntrState;
3494}
3495
3496
3497/**
3498 * Exports the guest's interruptibility-state into the guest-state area in the
3499 * VMCS.
3500 *
3501 * @returns VBox status code.
3502 * @param pVCpu The cross context virtual CPU structure.
3503 * @param fIntrState The interruptibility-state to set.
3504 */
3505static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3506{
3507 NOREF(pVCpu);
3508 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3509 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3510 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3511}
3512
3513
3514/**
3515 * Exports the exception intercepts required for guest execution in the VMCS.
3516 *
3517 * @returns VBox status code.
3518 * @param pVCpu The cross context virtual CPU structure.
3519 *
3520 * @remarks No-long-jump zone!!!
3521 */
3522static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3523{
3524 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3525 {
3526 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3527 if (pVCpu->hm.s.fGIMTrapXcptUD)
3528 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3529#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530 else
3531 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3532#endif
3533
3534 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3535 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3536
3537 /** @todo Optimize by checking cache before writing to VMCS. */
3538 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3539 AssertRCReturn(rc, rc);
3540
3541 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3542 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3543 }
3544 return VINF_SUCCESS;
3545}
3546
3547
3548/**
3549 * Exports the guest's RIP into the guest-state area in the VMCS.
3550 *
3551 * @returns VBox status code.
3552 * @param pVCpu The cross context virtual CPU structure.
3553 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3554 * out-of-sync. Make sure to update the required fields
3555 * before using them.
3556 *
3557 * @remarks No-long-jump zone!!!
3558 */
3559static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3560{
3561 int rc = VINF_SUCCESS;
3562 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3563 {
3564 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RIP);
3565
3566 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3567 AssertRCReturn(rc, rc);
3568
3569 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3570 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3571 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3572 else
3573 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3574
3575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3576 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3577 }
3578 return rc;
3579}
3580
3581
3582/**
3583 * Exports the guest's RSP into the guest-state area in the VMCS.
3584 *
3585 * @returns VBox status code.
3586 * @param pVCpu The cross context virtual CPU structure.
3587 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3588 * out-of-sync. Make sure to update the required fields
3589 * before using them.
3590 *
3591 * @remarks No-long-jump zone!!!
3592 */
3593static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3594{
3595 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3596 {
3597 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RSP);
3598
3599 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3600 AssertRCReturn(rc, rc);
3601
3602 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3603 }
3604 return VINF_SUCCESS;
3605}
3606
3607
3608/**
3609 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3610 *
3611 * @returns VBox status code.
3612 * @param pVCpu The cross context virtual CPU structure.
3613 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3614 * out-of-sync. Make sure to update the required fields
3615 * before using them.
3616 *
3617 * @remarks No-long-jump zone!!!
3618 */
3619static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3620{
3621 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3622 {
3623 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_RFLAGS);
3624
3625 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3626 Let us assert it as such and use 32-bit VMWRITE. */
3627 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3628 X86EFLAGS fEFlags = pMixedCtx->eflags;
3629 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3630 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3631
3632 /*
3633 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3634 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3635 * can run the real-mode guest code under Virtual 8086 mode.
3636 */
3637 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3638 {
3639 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3640 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3641 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3642 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3643 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3644 }
3645
3646 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3647 AssertRCReturn(rc, rc);
3648
3649 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3650 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3651 }
3652 return VINF_SUCCESS;
3653}
3654
3655
3656/**
3657 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3658 *
3659 * The guest FPU state is always pre-loaded hence we don't need to bother about
3660 * sharing FPU related CR0 bits between the guest and host.
3661 *
3662 * @returns VBox status code.
3663 * @param pVCpu The cross context virtual CPU structure.
3664 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3665 * out-of-sync. Make sure to update the required fields
3666 * before using them.
3667 *
3668 * @remarks No-long-jump zone!!!
3669 */
3670static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3671{
3672 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3673 {
3674 PVM pVM = pVCpu->CTX_SUFF(pVM);
3675 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR0);
3676 Assert(!RT_HI_U32(pMixedCtx->cr0));
3677
3678 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3679 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3680
3681 /*
3682 * Setup VT-x's view of the guest CR0.
3683 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3684 */
3685 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3686 if (pVM->hm.s.fNestedPaging)
3687 {
3688 if (CPUMIsGuestPagingEnabled(pVCpu))
3689 {
3690 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3691 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3692 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3693 }
3694 else
3695 {
3696 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3697 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3698 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3699 }
3700
3701 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3702 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3703 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3704 }
3705 else
3706 {
3707 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3708 u32GuestCr0 |= X86_CR0_WP;
3709 }
3710
3711 /*
3712 * Guest FPU bits.
3713 *
3714 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3715 * using CR0.TS.
3716 *
3717 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3718 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3719 */
3720 u32GuestCr0 |= X86_CR0_NE;
3721
3722 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3723 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3724
3725 /*
3726 * Update exception intercepts.
3727 */
3728 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3729 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3730 {
3731 Assert(PDMVmmDevHeapIsEnabled(pVM));
3732 Assert(pVM->hm.s.vmx.pRealModeTSS);
3733 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3734 }
3735 else
3736 {
3737 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3738 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3739 if (fInterceptMF)
3740 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3741 }
3742
3743 /* Additional intercepts for debugging, define these yourself explicitly. */
3744#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3745 uXcptBitmap |= 0
3746 | RT_BIT(X86_XCPT_BP)
3747 | RT_BIT(X86_XCPT_DE)
3748 | RT_BIT(X86_XCPT_NM)
3749 | RT_BIT(X86_XCPT_TS)
3750 | RT_BIT(X86_XCPT_UD)
3751 | RT_BIT(X86_XCPT_NP)
3752 | RT_BIT(X86_XCPT_SS)
3753 | RT_BIT(X86_XCPT_GP)
3754 | RT_BIT(X86_XCPT_PF)
3755 | RT_BIT(X86_XCPT_MF)
3756 ;
3757#elif defined(HMVMX_ALWAYS_TRAP_PF)
3758 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3759#endif
3760 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3761 {
3762 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3763 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3764 }
3765 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3766
3767 /*
3768 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3769 */
3770 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3771 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3772 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3773 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3774 else
3775 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3776
3777 u32GuestCr0 |= fSetCr0;
3778 u32GuestCr0 &= fZapCr0;
3779 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3780
3781 /*
3782 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3783 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3784 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3785 */
3786 uint32_t u32Cr0Mask = X86_CR0_PE
3787 | X86_CR0_NE
3788 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3789 | X86_CR0_PG
3790 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3791 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3792 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3793
3794 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3795 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3796 * and @bugref{6944}. */
3797#if 0
3798 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3799 u32Cr0Mask &= ~X86_CR0_PE;
3800#endif
3801 /*
3802 * Finally, update VMCS fields with the CR0 values.
3803 */
3804 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3805 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3806 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3807 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3808 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3809 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3810 AssertRCReturn(rc, rc);
3811
3812 /* Update our caches. */
3813 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3814 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3815
3816 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3817
3818 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3819 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3820 }
3821
3822 return VINF_SUCCESS;
3823}
3824
3825
3826/**
3827 * Exports the guest control registers (CR3, CR4) into the guest-state area
3828 * in the VMCS.
3829 *
3830 * @returns VBox strict status code.
3831 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3832 * without unrestricted guest access and the VMMDev is not presently
3833 * mapped (e.g. EFI32).
3834 *
3835 * @param pVCpu The cross context virtual CPU structure.
3836 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3837 * out-of-sync. Make sure to update the required fields
3838 * before using them.
3839 *
3840 * @remarks No-long-jump zone!!!
3841 */
3842static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3843{
3844 int rc = VINF_SUCCESS;
3845 PVM pVM = pVCpu->CTX_SUFF(pVM);
3846
3847 /*
3848 * Guest CR2.
3849 * It's always loaded in the assembler code. Nothing to do here.
3850 */
3851
3852 /*
3853 * Guest CR3.
3854 */
3855 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3856 {
3857 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR3);
3858
3859 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3860 if (pVM->hm.s.fNestedPaging)
3861 {
3862 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3863
3864 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3865 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3866 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3867 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3868
3869 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3870 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3871 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3872
3873 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3874 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3875 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3876 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3877 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3878 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3879 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3880
3881 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3882 AssertRCReturn(rc, rc);
3883
3884 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3885 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3886 {
3887 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3888 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3889 {
3890 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3891 AssertRCReturn(rc, rc);
3892 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3893 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3894 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3895 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3896 AssertRCReturn(rc, rc);
3897 }
3898
3899 /*
3900 * The guest's view of its CR3 is unblemished with Nested Paging when the
3901 * guest is using paging or we have unrestricted guest execution to handle
3902 * the guest when it's not using paging.
3903 */
3904 GCPhysGuestCR3 = pMixedCtx->cr3;
3905 }
3906 else
3907 {
3908 /*
3909 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3910 * thinks it accesses physical memory directly, we use our identity-mapped
3911 * page table to map guest-linear to guest-physical addresses. EPT takes care
3912 * of translating it to host-physical addresses.
3913 */
3914 RTGCPHYS GCPhys;
3915 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3916
3917 /* We obtain it here every time as the guest could have relocated this PCI region. */
3918 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3919 if (RT_SUCCESS(rc))
3920 { /* likely */ }
3921 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3922 {
3923 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3924 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3925 }
3926 else
3927 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3928
3929 GCPhysGuestCR3 = GCPhys;
3930 }
3931
3932 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3933 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3934 AssertRCReturn(rc, rc);
3935 }
3936 else
3937 {
3938 /* Non-nested paging case, just use the hypervisor's CR3. */
3939 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3940
3941 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3942 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3943 AssertRCReturn(rc, rc);
3944 }
3945
3946 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3947 }
3948
3949 /*
3950 * Guest CR4.
3951 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3952 */
3953 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3954 {
3955 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CR4);
3956 Assert(!RT_HI_U32(pMixedCtx->cr4));
3957
3958 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3959 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3960
3961 /*
3962 * Setup VT-x's view of the guest CR4.
3963 *
3964 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3965 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3966 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3967 *
3968 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3969 */
3970 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3971 {
3972 Assert(pVM->hm.s.vmx.pRealModeTSS);
3973 Assert(PDMVmmDevHeapIsEnabled(pVM));
3974 u32GuestCr4 &= ~X86_CR4_VME;
3975 }
3976
3977 if (pVM->hm.s.fNestedPaging)
3978 {
3979 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3980 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3981 {
3982 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3983 u32GuestCr4 |= X86_CR4_PSE;
3984 /* Our identity mapping is a 32-bit page directory. */
3985 u32GuestCr4 &= ~X86_CR4_PAE;
3986 }
3987 /* else use guest CR4.*/
3988 }
3989 else
3990 {
3991 /*
3992 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3993 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3994 */
3995 switch (pVCpu->hm.s.enmShadowMode)
3996 {
3997 case PGMMODE_REAL: /* Real-mode. */
3998 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3999 case PGMMODE_32_BIT: /* 32-bit paging. */
4000 {
4001 u32GuestCr4 &= ~X86_CR4_PAE;
4002 break;
4003 }
4004
4005 case PGMMODE_PAE: /* PAE paging. */
4006 case PGMMODE_PAE_NX: /* PAE paging with NX. */
4007 {
4008 u32GuestCr4 |= X86_CR4_PAE;
4009 break;
4010 }
4011
4012 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4013 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4014#ifdef VBOX_ENABLE_64_BITS_GUESTS
4015 break;
4016#endif
4017 default:
4018 AssertFailed();
4019 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4020 }
4021 }
4022
4023 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4024 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4025 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4026 u32GuestCr4 |= fSetCr4;
4027 u32GuestCr4 &= fZapCr4;
4028
4029 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4030 that would cause a VM-exit. */
4031 uint32_t u32Cr4Mask = X86_CR4_VME
4032 | X86_CR4_PAE
4033 | X86_CR4_PGE
4034 | X86_CR4_PSE
4035 | X86_CR4_VMXE;
4036 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4037 u32Cr4Mask |= X86_CR4_OSXSAVE;
4038 if (pVM->cpum.ro.GuestFeatures.fPcid)
4039 u32Cr4Mask |= X86_CR4_PCIDE;
4040
4041 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4042 into the VMCS and update our cache. */
4043 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4044 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4045 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4046 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4047 AssertRCReturn(rc, rc);
4048 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4049
4050 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4051 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4052
4053 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4054
4055 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4056 fZapCr4));
4057 }
4058 return rc;
4059}
4060
4061
4062/**
4063 * Exports the guest debug registers into the guest-state area in the VMCS.
4064 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4065 *
4066 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4067 *
4068 * @returns VBox status code.
4069 * @param pVCpu The cross context virtual CPU structure.
4070 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4071 * out-of-sync. Make sure to update the required fields
4072 * before using them.
4073 *
4074 * @remarks No-long-jump zone!!!
4075 */
4076static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4077{
4078 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4079
4080#ifdef VBOX_STRICT
4081 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4082 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4083 {
4084 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4085 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4086 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4087 }
4088#endif
4089
4090 bool fSteppingDB = false;
4091 bool fInterceptMovDRx = false;
4092 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4093 if (pVCpu->hm.s.fSingleInstruction)
4094 {
4095 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4096 PVM pVM = pVCpu->CTX_SUFF(pVM);
4097 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4098 {
4099 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4100 Assert(fSteppingDB == false);
4101 }
4102 else
4103 {
4104 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4105 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4106 pVCpu->hm.s.fClearTrapFlag = true;
4107 fSteppingDB = true;
4108 }
4109 }
4110
4111 uint32_t u32GuestDr7;
4112 if ( fSteppingDB
4113 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4114 {
4115 /*
4116 * Use the combined guest and host DRx values found in the hypervisor register set
4117 * because the debugger has breakpoints active or someone is single stepping on the
4118 * host side without a monitor trap flag.
4119 *
4120 * Note! DBGF expects a clean DR6 state before executing guest code.
4121 */
4122#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4123 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4124 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4125 {
4126 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4127 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4128 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4129 }
4130 else
4131#endif
4132 if (!CPUMIsHyperDebugStateActive(pVCpu))
4133 {
4134 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4135 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4136 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4137 }
4138
4139 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4140 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4141 pVCpu->hm.s.fUsingHyperDR7 = true;
4142 fInterceptMovDRx = true;
4143 }
4144 else
4145 {
4146 /*
4147 * If the guest has enabled debug registers, we need to load them prior to
4148 * executing guest code so they'll trigger at the right time.
4149 */
4150 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4151 {
4152#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4153 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4154 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4155 {
4156 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4157 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4158 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4159 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4160 }
4161 else
4162#endif
4163 if (!CPUMIsGuestDebugStateActive(pVCpu))
4164 {
4165 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4166 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4167 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4168 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4169 }
4170 Assert(!fInterceptMovDRx);
4171 }
4172 /*
4173 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4174 * must intercept #DB in order to maintain a correct DR6 guest value, and
4175 * because we need to intercept it to prevent nested #DBs from hanging the
4176 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4177 */
4178#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4179 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4180 && !CPUMIsGuestDebugStateActive(pVCpu))
4181#else
4182 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4183#endif
4184 {
4185 fInterceptMovDRx = true;
4186 }
4187
4188 /* Update DR7 with the actual guest value. */
4189 u32GuestDr7 = pMixedCtx->dr[7];
4190 pVCpu->hm.s.fUsingHyperDR7 = false;
4191 }
4192
4193 if (fInterceptMovDRx)
4194 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4195 else
4196 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4197
4198 /*
4199 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4200 * monitor-trap flag and update our cache.
4201 */
4202 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4203 {
4204 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4205 AssertRCReturn(rc2, rc2);
4206 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4207 }
4208
4209 /*
4210 * Update guest DR7.
4211 */
4212 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4213 AssertRCReturn(rc, rc);
4214
4215 return VINF_SUCCESS;
4216}
4217
4218
4219#ifdef VBOX_STRICT
4220/**
4221 * Strict function to validate segment registers.
4222 *
4223 * @param pVCpu The cross context virtual CPU structure.
4224 * @param pCtx Pointer to the guest-CPU context.
4225 *
4226 * @remarks Will import guest CR0 on strict builds during validation of
4227 * segments.
4228 */
4229static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4230{
4231 /*
4232 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4233 *
4234 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4235 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4236 * and doesn't change the guest-context value.
4237 */
4238 PVM pVM = pVCpu->CTX_SUFF(pVM);
4239 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4240 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4241 && ( !CPUMIsGuestInRealModeEx(pCtx)
4242 && !CPUMIsGuestInV86ModeEx(pCtx)))
4243 {
4244 /* Protected mode checks */
4245 /* CS */
4246 Assert(pCtx->cs.Attr.n.u1Present);
4247 Assert(!(pCtx->cs.Attr.u & 0xf00));
4248 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4249 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4250 || !(pCtx->cs.Attr.n.u1Granularity));
4251 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4252 || (pCtx->cs.Attr.n.u1Granularity));
4253 /* CS cannot be loaded with NULL in protected mode. */
4254 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4255 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4256 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4257 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4258 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4259 else
4260 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4261 /* SS */
4262 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4263 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4264 if ( !(pCtx->cr0 & X86_CR0_PE)
4265 || pCtx->cs.Attr.n.u4Type == 3)
4266 {
4267 Assert(!pCtx->ss.Attr.n.u2Dpl);
4268 }
4269 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4270 {
4271 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4272 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4273 Assert(pCtx->ss.Attr.n.u1Present);
4274 Assert(!(pCtx->ss.Attr.u & 0xf00));
4275 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4276 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4277 || !(pCtx->ss.Attr.n.u1Granularity));
4278 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4279 || (pCtx->ss.Attr.n.u1Granularity));
4280 }
4281 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4282 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4283 {
4284 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4285 Assert(pCtx->ds.Attr.n.u1Present);
4286 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4287 Assert(!(pCtx->ds.Attr.u & 0xf00));
4288 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4289 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4290 || !(pCtx->ds.Attr.n.u1Granularity));
4291 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4292 || (pCtx->ds.Attr.n.u1Granularity));
4293 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4294 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4295 }
4296 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4297 {
4298 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4299 Assert(pCtx->es.Attr.n.u1Present);
4300 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4301 Assert(!(pCtx->es.Attr.u & 0xf00));
4302 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4303 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4304 || !(pCtx->es.Attr.n.u1Granularity));
4305 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4306 || (pCtx->es.Attr.n.u1Granularity));
4307 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4308 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4309 }
4310 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4311 {
4312 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4313 Assert(pCtx->fs.Attr.n.u1Present);
4314 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4315 Assert(!(pCtx->fs.Attr.u & 0xf00));
4316 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4317 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4318 || !(pCtx->fs.Attr.n.u1Granularity));
4319 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4320 || (pCtx->fs.Attr.n.u1Granularity));
4321 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4322 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4323 }
4324 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4325 {
4326 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4327 Assert(pCtx->gs.Attr.n.u1Present);
4328 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4329 Assert(!(pCtx->gs.Attr.u & 0xf00));
4330 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4331 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4332 || !(pCtx->gs.Attr.n.u1Granularity));
4333 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4334 || (pCtx->gs.Attr.n.u1Granularity));
4335 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4336 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4337 }
4338 /* 64-bit capable CPUs. */
4339# if HC_ARCH_BITS == 64
4340 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4341 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4342 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4343 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4344# endif
4345 }
4346 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4347 || ( CPUMIsGuestInRealModeEx(pCtx)
4348 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4349 {
4350 /* Real and v86 mode checks. */
4351 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4352 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4353 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4354 {
4355 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4356 }
4357 else
4358 {
4359 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4360 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4361 }
4362
4363 /* CS */
4364 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4365 Assert(pCtx->cs.u32Limit == 0xffff);
4366 Assert(u32CSAttr == 0xf3);
4367 /* SS */
4368 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4369 Assert(pCtx->ss.u32Limit == 0xffff);
4370 Assert(u32SSAttr == 0xf3);
4371 /* DS */
4372 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4373 Assert(pCtx->ds.u32Limit == 0xffff);
4374 Assert(u32DSAttr == 0xf3);
4375 /* ES */
4376 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4377 Assert(pCtx->es.u32Limit == 0xffff);
4378 Assert(u32ESAttr == 0xf3);
4379 /* FS */
4380 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4381 Assert(pCtx->fs.u32Limit == 0xffff);
4382 Assert(u32FSAttr == 0xf3);
4383 /* GS */
4384 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4385 Assert(pCtx->gs.u32Limit == 0xffff);
4386 Assert(u32GSAttr == 0xf3);
4387 /* 64-bit capable CPUs. */
4388# if HC_ARCH_BITS == 64
4389 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4390 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4391 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4392 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4393# endif
4394 }
4395}
4396#endif /* VBOX_STRICT */
4397
4398
4399/**
4400 * Exports a guest segment register into the guest-state area in the VMCS.
4401 *
4402 * @returns VBox status code.
4403 * @param pVCpu The cross context virtual CPU structure.
4404 * @param idxSel Index of the selector in the VMCS.
4405 * @param idxLimit Index of the segment limit in the VMCS.
4406 * @param idxBase Index of the segment base in the VMCS.
4407 * @param idxAccess Index of the access rights of the segment in the VMCS.
4408 * @param pSelReg Pointer to the segment selector.
4409 *
4410 * @remarks No-long-jump zone!!!
4411 */
4412static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4413 PCCPUMSELREG pSelReg)
4414{
4415 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4416 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4417 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4418 AssertRCReturn(rc, rc);
4419
4420 uint32_t u32Access = pSelReg->Attr.u;
4421 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4422 {
4423 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4424 u32Access = 0xf3;
4425 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4426 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4427 }
4428 else
4429 {
4430 /*
4431 * The way to differentiate between whether this is really a null selector or was just
4432 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4433 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4434 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4435 * NULL selectors loaded in protected-mode have their attribute as 0.
4436 */
4437 if (!u32Access)
4438 u32Access = X86DESCATTR_UNUSABLE;
4439 }
4440
4441 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4442 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4443 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4444
4445 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4446 AssertRCReturn(rc, rc);
4447 return rc;
4448}
4449
4450
4451/**
4452 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4453 * into the guest-state area in the VMCS.
4454 *
4455 * @returns VBox status code.
4456 * @param pVCpu The cross context virtual CPU structure.
4457 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4458 * out-of-sync. Make sure to update the required fields
4459 * before using them.
4460 *
4461 * @remarks Will import guest CR0 on strict builds during validation of
4462 * segments.
4463 * @remarks No-long-jump zone!!!
4464 */
4465static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4466{
4467 int rc = VERR_INTERNAL_ERROR_5;
4468 PVM pVM = pVCpu->CTX_SUFF(pVM);
4469
4470 /*
4471 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4472 */
4473 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4474 {
4475#ifdef VBOX_WITH_REM
4476 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4477 {
4478 Assert(pVM->hm.s.vmx.pRealModeTSS);
4479 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4480 if ( pVCpu->hm.s.vmx.fWasInRealMode
4481 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4482 {
4483 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4484 in real-mode (e.g. OpenBSD 4.0) */
4485 REMFlushTBs(pVM);
4486 Log4Func(("Switch to protected mode detected!\n"));
4487 pVCpu->hm.s.vmx.fWasInRealMode = false;
4488 }
4489 }
4490#endif
4491 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4492 {
4493 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_CS);
4494 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4495 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4496 rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4497 AssertRCReturn(rc, rc);
4498 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4499 }
4500
4501 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4502 {
4503 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SS);
4504 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4505 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4506 rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4507 AssertRCReturn(rc, rc);
4508 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4509 }
4510
4511 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4512 {
4513 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_DS);
4514 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4515 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4516 rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4517 AssertRCReturn(rc, rc);
4518 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4519 }
4520
4521 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4522 {
4523 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_ES);
4524 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4525 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4526 rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4527 AssertRCReturn(rc, rc);
4528 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4529 }
4530
4531 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4532 {
4533 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_FS);
4534 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4535 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4536 rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4537 AssertRCReturn(rc, rc);
4538 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4539 }
4540
4541 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4542 {
4543 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GS);
4544 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4545 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4546 rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4547 AssertRCReturn(rc, rc);
4548 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4549 }
4550
4551#ifdef VBOX_STRICT
4552 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4553#endif
4554
4555 /* Update the exit history entry with the correct CS.BASE + RIP. */
4556 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4557 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4558
4559 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4560 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4561 }
4562
4563 /*
4564 * Guest TR.
4565 */
4566 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4567 {
4568 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_TR);
4569
4570 /*
4571 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4572 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4573 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4574 */
4575 uint16_t u16Sel = 0;
4576 uint32_t u32Limit = 0;
4577 uint64_t u64Base = 0;
4578 uint32_t u32AccessRights = 0;
4579
4580 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4581 {
4582 u16Sel = pMixedCtx->tr.Sel;
4583 u32Limit = pMixedCtx->tr.u32Limit;
4584 u64Base = pMixedCtx->tr.u64Base;
4585 u32AccessRights = pMixedCtx->tr.Attr.u;
4586 }
4587 else
4588 {
4589 Assert(pVM->hm.s.vmx.pRealModeTSS);
4590 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4591
4592 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4593 RTGCPHYS GCPhys;
4594 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4595 AssertRCReturn(rc, rc);
4596
4597 X86DESCATTR DescAttr;
4598 DescAttr.u = 0;
4599 DescAttr.n.u1Present = 1;
4600 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4601
4602 u16Sel = 0;
4603 u32Limit = HM_VTX_TSS_SIZE;
4604 u64Base = GCPhys; /* in real-mode phys = virt. */
4605 u32AccessRights = DescAttr.u;
4606 }
4607
4608 /* Validate. */
4609 Assert(!(u16Sel & RT_BIT(2)));
4610 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4611 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4612 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4613 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4614 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4615 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4616 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4617 Assert( (u32Limit & 0xfff) == 0xfff
4618 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4619 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4620 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4621
4622 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4623 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4624 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4625 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4626 AssertRCReturn(rc, rc);
4627
4628 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4629 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4630 }
4631
4632 /*
4633 * Guest GDTR.
4634 */
4635 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4636 {
4637 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_GDTR);
4638
4639 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4640 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4641 AssertRCReturn(rc, rc);
4642
4643 /* Validate. */
4644 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4645
4646 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4647 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4648 }
4649
4650 /*
4651 * Guest LDTR.
4652 */
4653 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4654 {
4655 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_LDTR);
4656
4657 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4658 uint32_t u32Access = 0;
4659 if (!pMixedCtx->ldtr.Attr.u)
4660 u32Access = X86DESCATTR_UNUSABLE;
4661 else
4662 u32Access = pMixedCtx->ldtr.Attr.u;
4663
4664 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4665 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4666 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4667 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4668 AssertRCReturn(rc, rc);
4669
4670 /* Validate. */
4671 if (!(u32Access & X86DESCATTR_UNUSABLE))
4672 {
4673 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4674 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4675 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4676 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4677 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4678 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4679 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4680 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4681 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4682 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4683 }
4684
4685 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4686 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4687 }
4688
4689 /*
4690 * Guest IDTR.
4691 */
4692 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4693 {
4694 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_IDTR);
4695
4696 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4697 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4698 AssertRCReturn(rc, rc);
4699
4700 /* Validate. */
4701 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4702
4703 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4704 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4705 }
4706
4707 return VINF_SUCCESS;
4708}
4709
4710
4711/**
4712 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4713 * areas.
4714 *
4715 * These MSRs will automatically be loaded to the host CPU on every successful
4716 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4717 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4718 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4719 *
4720 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4721 *
4722 * @returns VBox status code.
4723 * @param pVCpu The cross context virtual CPU structure.
4724 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4725 * out-of-sync. Make sure to update the required fields
4726 * before using them.
4727 *
4728 * @remarks No-long-jump zone!!!
4729 */
4730static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4731{
4732 AssertPtr(pVCpu);
4733 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4734
4735 /*
4736 * MSRs that we use the auto-load/store MSR area in the VMCS.
4737 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4738 */
4739 PVM pVM = pVCpu->CTX_SUFF(pVM);
4740 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4741 {
4742 if (pVM->hm.s.fAllow64BitGuests)
4743 {
4744#if HC_ARCH_BITS == 32
4745 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSCALL_MSRS | CPUMCTX_EXTRN_KERNEL_GS_BASE);
4746
4747 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4748 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4749 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4750 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4751 AssertRCReturn(rc, rc);
4752# ifdef LOG_ENABLED
4753 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4754 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4755 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4756# endif
4757#endif
4758 }
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4760 }
4761
4762 /*
4763 * Guest Sysenter MSRs.
4764 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4765 * VM-exits on WRMSRs for these MSRs.
4766 */
4767 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4768 {
4769 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_SYSENTER_MSRS);
4770
4771 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4772 {
4773 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4774 AssertRCReturn(rc, rc);
4775 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4776 }
4777
4778 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4779 {
4780 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4781 AssertRCReturn(rc, rc);
4782 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4783 }
4784
4785 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4786 {
4787 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4788 AssertRCReturn(rc, rc);
4789 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4790 }
4791 }
4792
4793 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4794 {
4795 HMVMX_CPUMCTX_ASSERT(pVCpu, CPUMCTX_EXTRN_EFER);
4796
4797 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4798 {
4799 /*
4800 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4801 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4802 */
4803 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4804 {
4805 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4806 AssertRCReturn(rc,rc);
4807 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4808 }
4809 else
4810 {
4811 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4812 NULL /* pfAddedAndUpdated */);
4813 AssertRCReturn(rc, rc);
4814
4815 /* We need to intercept reads too, see @bugref{7386#c16}. */
4816 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4817 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4818 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4819 pVCpu->hm.s.vmx.cMsrs));
4820 }
4821 }
4822 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4823 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4824 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4825 }
4826
4827 return VINF_SUCCESS;
4828}
4829
4830
4831#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4832/**
4833 * Check if guest state allows safe use of 32-bit switcher again.
4834 *
4835 * Segment bases and protected mode structures must be 32-bit addressable
4836 * because the 32-bit switcher will ignore high dword when writing these VMCS
4837 * fields. See @bugref{8432} for details.
4838 *
4839 * @returns true if safe, false if must continue to use the 64-bit switcher.
4840 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4841 * out-of-sync. Make sure to update the required fields
4842 * before using them.
4843 *
4844 * @remarks No-long-jump zone!!!
4845 */
4846static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4847{
4848 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4849 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4850 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4851 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4852 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4853 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4854 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4855 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4856 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4857 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4858
4859 /* All good, bases are 32-bit. */
4860 return true;
4861}
4862#endif
4863
4864
4865/**
4866 * Selects up the appropriate function to run guest code.
4867 *
4868 * @returns VBox status code.
4869 * @param pVCpu The cross context virtual CPU structure.
4870 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4871 * out-of-sync. Make sure to update the required fields
4872 * before using them.
4873 *
4874 * @remarks No-long-jump zone!!!
4875 */
4876static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4877{
4878 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4879 {
4880#ifndef VBOX_ENABLE_64_BITS_GUESTS
4881 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4882#endif
4883 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4884#if HC_ARCH_BITS == 32
4885 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4886 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4887 {
4888#ifdef VBOX_STRICT
4889 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4890 {
4891 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4892 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4893 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4894 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4895 | HM_CHANGED_VMX_ENTRY_CTLS
4896 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4897 }
4898#endif
4899 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4900
4901 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4902 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4903 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4904 Log4Func(("Selected 64-bit switcher\n"));
4905 }
4906#else
4907 /* 64-bit host. */
4908 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4909#endif
4910 }
4911 else
4912 {
4913 /* Guest is not in long mode, use the 32-bit handler. */
4914#if HC_ARCH_BITS == 32
4915 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4916 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4917 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4918 {
4919# ifdef VBOX_STRICT
4920 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4921 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4922 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4923 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4924 | HM_CHANGED_VMX_ENTRY_CTLS
4925 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4926# endif
4927 }
4928# ifdef VBOX_ENABLE_64_BITS_GUESTS
4929 /*
4930 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4931 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4932 * switcher flag because now we know the guest is in a sane state where it's safe
4933 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4934 * the much faster 32-bit switcher again.
4935 */
4936 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4937 {
4938 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4939 Log4Func(("Selected 32-bit switcher\n"));
4940 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4941 }
4942 else
4943 {
4944 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4945 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4946 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4947 {
4948 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4949 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4950 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4951 | HM_CHANGED_VMX_ENTRY_CTLS
4952 | HM_CHANGED_VMX_EXIT_CTLS
4953 | HM_CHANGED_HOST_CONTEXT);
4954 Log4Func(("Selected 32-bit switcher (safe)\n"));
4955 }
4956 }
4957# else
4958 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4959# endif
4960#else
4961 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4962#endif
4963 }
4964 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4965 return VINF_SUCCESS;
4966}
4967
4968
4969/**
4970 * Wrapper for running the guest code in VT-x.
4971 *
4972 * @returns VBox status code, no informational status codes.
4973 * @param pVCpu The cross context virtual CPU structure.
4974 * @param pCtx Pointer to the guest-CPU context.
4975 *
4976 * @remarks No-long-jump zone!!!
4977 */
4978DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4979{
4980 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4981 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4982
4983 /*
4984 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4985 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4986 * callee-saved and thus the need for this XMM wrapper.
4987 *
4988 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4989 */
4990 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4991 /** @todo Add stats for resume vs launch. */
4992 PVM pVM = pVCpu->CTX_SUFF(pVM);
4993#ifdef VBOX_WITH_KERNEL_USING_XMM
4994 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4995#else
4996 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4997#endif
4998 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4999 return rc;
5000}
5001
5002
5003/**
5004 * Reports world-switch error and dumps some useful debug info.
5005 *
5006 * @param pVCpu The cross context virtual CPU structure.
5007 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
5008 * @param pCtx Pointer to the guest-CPU context.
5009 * @param pVmxTransient Pointer to the VMX transient structure (only
5010 * exitReason updated).
5011 */
5012static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
5013{
5014 Assert(pVCpu);
5015 Assert(pCtx);
5016 Assert(pVmxTransient);
5017 HMVMX_ASSERT_PREEMPT_SAFE();
5018
5019 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
5020 switch (rcVMRun)
5021 {
5022 case VERR_VMX_INVALID_VMXON_PTR:
5023 AssertFailed();
5024 break;
5025 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
5026 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
5027 {
5028 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
5029 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
5030 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
5031 AssertRC(rc);
5032
5033 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5034 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5035 Cannot do it here as we may have been long preempted. */
5036
5037#ifdef VBOX_STRICT
5038 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5039 pVmxTransient->uExitReason));
5040 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5041 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5042 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5043 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5044 else
5045 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5046 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5047 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5048
5049 /* VMX control bits. */
5050 uint32_t u32Val;
5051 uint64_t u64Val;
5052 RTHCUINTREG uHCReg;
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5055 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5056 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5057 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5058 {
5059 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5060 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5061 }
5062 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5063 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5064 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5065 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5066 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5067 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5068 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5069 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5070 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5071 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5072 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5073 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5074 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5075 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5076 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5077 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5078 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5080 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5081 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5082 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5083 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5084 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5085 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5086 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5087 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5088 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5089 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5090 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5091 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5092 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5093 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5094 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5095 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5096 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5097 {
5098 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5099 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5100 }
5101
5102 /* Guest bits. */
5103 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5104 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5105 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5106 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5107 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5108 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5109 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5110 {
5111 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5112 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5113 }
5114
5115 /* Host bits. */
5116 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5117 Log4(("Host CR0 %#RHr\n", uHCReg));
5118 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5119 Log4(("Host CR3 %#RHr\n", uHCReg));
5120 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5121 Log4(("Host CR4 %#RHr\n", uHCReg));
5122
5123 RTGDTR HostGdtr;
5124 PCX86DESCHC pDesc;
5125 ASMGetGDTR(&HostGdtr);
5126 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5127 Log4(("Host CS %#08x\n", u32Val));
5128 if (u32Val < HostGdtr.cbGdt)
5129 {
5130 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5131 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5132 }
5133
5134 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5135 Log4(("Host DS %#08x\n", u32Val));
5136 if (u32Val < HostGdtr.cbGdt)
5137 {
5138 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5139 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5140 }
5141
5142 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5143 Log4(("Host ES %#08x\n", u32Val));
5144 if (u32Val < HostGdtr.cbGdt)
5145 {
5146 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5147 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5148 }
5149
5150 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5151 Log4(("Host FS %#08x\n", u32Val));
5152 if (u32Val < HostGdtr.cbGdt)
5153 {
5154 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5155 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5156 }
5157
5158 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5159 Log4(("Host GS %#08x\n", u32Val));
5160 if (u32Val < HostGdtr.cbGdt)
5161 {
5162 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5163 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5164 }
5165
5166 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5167 Log4(("Host SS %#08x\n", u32Val));
5168 if (u32Val < HostGdtr.cbGdt)
5169 {
5170 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5171 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5172 }
5173
5174 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5175 Log4(("Host TR %#08x\n", u32Val));
5176 if (u32Val < HostGdtr.cbGdt)
5177 {
5178 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5179 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5180 }
5181
5182 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5183 Log4(("Host TR Base %#RHv\n", uHCReg));
5184 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5185 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5186 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5187 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5188 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5189 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5190 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5191 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5192 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5193 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5194 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5195 Log4(("Host RSP %#RHv\n", uHCReg));
5196 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5197 Log4(("Host RIP %#RHv\n", uHCReg));
5198# if HC_ARCH_BITS == 64
5199 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5200 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5201 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5202 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5203 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5204 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5205# endif
5206#endif /* VBOX_STRICT */
5207 break;
5208 }
5209
5210 default:
5211 /* Impossible */
5212 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5213 break;
5214 }
5215 NOREF(pCtx);
5216}
5217
5218
5219#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5220#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5221# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5222#endif
5223#ifdef VBOX_STRICT
5224static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5225{
5226 switch (idxField)
5227 {
5228 case VMX_VMCS_GUEST_RIP:
5229 case VMX_VMCS_GUEST_RSP:
5230 case VMX_VMCS_GUEST_SYSENTER_EIP:
5231 case VMX_VMCS_GUEST_SYSENTER_ESP:
5232 case VMX_VMCS_GUEST_GDTR_BASE:
5233 case VMX_VMCS_GUEST_IDTR_BASE:
5234 case VMX_VMCS_GUEST_CS_BASE:
5235 case VMX_VMCS_GUEST_DS_BASE:
5236 case VMX_VMCS_GUEST_ES_BASE:
5237 case VMX_VMCS_GUEST_FS_BASE:
5238 case VMX_VMCS_GUEST_GS_BASE:
5239 case VMX_VMCS_GUEST_SS_BASE:
5240 case VMX_VMCS_GUEST_LDTR_BASE:
5241 case VMX_VMCS_GUEST_TR_BASE:
5242 case VMX_VMCS_GUEST_CR3:
5243 return true;
5244 }
5245 return false;
5246}
5247
5248static bool hmR0VmxIsValidReadField(uint32_t idxField)
5249{
5250 switch (idxField)
5251 {
5252 /* Read-only fields. */
5253 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5254 return true;
5255 }
5256 /* Remaining readable fields should also be writable. */
5257 return hmR0VmxIsValidWriteField(idxField);
5258}
5259#endif /* VBOX_STRICT */
5260
5261
5262/**
5263 * Executes the specified handler in 64-bit mode.
5264 *
5265 * @returns VBox status code (no informational status codes).
5266 * @param pVCpu The cross context virtual CPU structure.
5267 * @param enmOp The operation to perform.
5268 * @param cParams Number of parameters.
5269 * @param paParam Array of 32-bit parameters.
5270 */
5271VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5272{
5273 PVM pVM = pVCpu->CTX_SUFF(pVM);
5274 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5275 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5276 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5277 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5278
5279#ifdef VBOX_STRICT
5280 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5281 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5282
5283 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5284 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5285#endif
5286
5287 /* Disable interrupts. */
5288 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5289
5290#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5291 RTCPUID idHostCpu = RTMpCpuId();
5292 CPUMR0SetLApic(pVCpu, idHostCpu);
5293#endif
5294
5295 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5296 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5297
5298 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5299 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5300 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5301
5302 /* Leave VMX Root Mode. */
5303 VMXDisable();
5304
5305 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5306
5307 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5308 CPUMSetHyperEIP(pVCpu, enmOp);
5309 for (int i = (int)cParams - 1; i >= 0; i--)
5310 CPUMPushHyper(pVCpu, paParam[i]);
5311
5312 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5313
5314 /* Call the switcher. */
5315 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5316 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5317
5318 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5319 /* Make sure the VMX instructions don't cause #UD faults. */
5320 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5321
5322 /* Re-enter VMX Root Mode */
5323 int rc2 = VMXEnable(HCPhysCpuPage);
5324 if (RT_FAILURE(rc2))
5325 {
5326 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5327 ASMSetFlags(fOldEFlags);
5328 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5329 return rc2;
5330 }
5331
5332 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5333 AssertRC(rc2);
5334 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5335 Assert(!(ASMGetFlags() & X86_EFL_IF));
5336 ASMSetFlags(fOldEFlags);
5337 return rc;
5338}
5339
5340
5341/**
5342 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5343 * supporting 64-bit guests.
5344 *
5345 * @returns VBox status code.
5346 * @param fResume Whether to VMLAUNCH or VMRESUME.
5347 * @param pCtx Pointer to the guest-CPU context.
5348 * @param pCache Pointer to the VMCS cache.
5349 * @param pVM The cross context VM structure.
5350 * @param pVCpu The cross context virtual CPU structure.
5351 */
5352DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5353{
5354 NOREF(fResume);
5355
5356 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5357 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5358
5359#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5360 pCache->uPos = 1;
5361 pCache->interPD = PGMGetInterPaeCR3(pVM);
5362 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5363#endif
5364
5365#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5366 pCache->TestIn.HCPhysCpuPage = 0;
5367 pCache->TestIn.HCPhysVmcs = 0;
5368 pCache->TestIn.pCache = 0;
5369 pCache->TestOut.HCPhysVmcs = 0;
5370 pCache->TestOut.pCache = 0;
5371 pCache->TestOut.pCtx = 0;
5372 pCache->TestOut.eflags = 0;
5373#else
5374 NOREF(pCache);
5375#endif
5376
5377 uint32_t aParam[10];
5378 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5379 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5380 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5381 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5382 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5383 aParam[5] = 0;
5384 aParam[6] = VM_RC_ADDR(pVM, pVM);
5385 aParam[7] = 0;
5386 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5387 aParam[9] = 0;
5388
5389#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5390 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5391 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5392#endif
5393 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5394
5395#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5396 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5397 Assert(pCtx->dr[4] == 10);
5398 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5399#endif
5400
5401#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5402 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5403 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5404 pVCpu->hm.s.vmx.HCPhysVmcs));
5405 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5406 pCache->TestOut.HCPhysVmcs));
5407 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5408 pCache->TestOut.pCache));
5409 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5410 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5411 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5412 pCache->TestOut.pCtx));
5413 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5414#endif
5415 NOREF(pCtx);
5416 return rc;
5417}
5418
5419
5420/**
5421 * Initialize the VMCS-Read cache.
5422 *
5423 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5424 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5425 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5426 * (those that have a 32-bit FULL & HIGH part).
5427 *
5428 * @returns VBox status code.
5429 * @param pVCpu The cross context virtual CPU structure.
5430 */
5431static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5432{
5433#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5434 do { \
5435 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5436 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5437 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5438 ++cReadFields; \
5439 } while (0)
5440
5441 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5442 uint32_t cReadFields = 0;
5443
5444 /*
5445 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5446 * and serve to indicate exceptions to the rules.
5447 */
5448
5449 /* Guest-natural selector base fields. */
5450#if 0
5451 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5454#endif
5455 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5456 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5458 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5459 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5461 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5462 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5463 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5464 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5466 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5467#if 0
5468 /* Unused natural width guest-state fields. */
5469 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5470 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5471#endif
5472 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5473 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5474
5475 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5476 these 64-bit fields (using "FULL" and "HIGH" fields). */
5477#if 0
5478 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5479 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5480 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5481 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5482 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5483 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5484 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5485 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5486 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5487#endif
5488
5489 /* Natural width guest-state fields. */
5490 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5491#if 0
5492 /* Currently unused field. */
5493 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5494#endif
5495
5496 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5497 {
5498 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5499 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5500 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5501 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5502 }
5503 else
5504 {
5505 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5506 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5507 }
5508
5509#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5510 return VINF_SUCCESS;
5511}
5512
5513
5514/**
5515 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5516 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5517 * darwin, running 64-bit guests).
5518 *
5519 * @returns VBox status code.
5520 * @param pVCpu The cross context virtual CPU structure.
5521 * @param idxField The VMCS field encoding.
5522 * @param u64Val 16, 32 or 64-bit value.
5523 */
5524VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5525{
5526 int rc;
5527 switch (idxField)
5528 {
5529 /*
5530 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5531 */
5532 /* 64-bit Control fields. */
5533 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5534 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5535 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5536 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5537 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5538 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5539 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5540 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5541 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5542 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5543 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5544 case VMX_VMCS64_CTRL_EPTP_FULL:
5545 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5546 /* 64-bit Guest-state fields. */
5547 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5548 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5549 case VMX_VMCS64_GUEST_PAT_FULL:
5550 case VMX_VMCS64_GUEST_EFER_FULL:
5551 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5552 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5553 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5554 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5555 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5556 /* 64-bit Host-state fields. */
5557 case VMX_VMCS64_HOST_PAT_FULL:
5558 case VMX_VMCS64_HOST_EFER_FULL:
5559 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5560 {
5561 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5562 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5563 break;
5564 }
5565
5566 /*
5567 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5568 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5569 */
5570 /* Natural-width Guest-state fields. */
5571 case VMX_VMCS_GUEST_CR3:
5572 case VMX_VMCS_GUEST_ES_BASE:
5573 case VMX_VMCS_GUEST_CS_BASE:
5574 case VMX_VMCS_GUEST_SS_BASE:
5575 case VMX_VMCS_GUEST_DS_BASE:
5576 case VMX_VMCS_GUEST_FS_BASE:
5577 case VMX_VMCS_GUEST_GS_BASE:
5578 case VMX_VMCS_GUEST_LDTR_BASE:
5579 case VMX_VMCS_GUEST_TR_BASE:
5580 case VMX_VMCS_GUEST_GDTR_BASE:
5581 case VMX_VMCS_GUEST_IDTR_BASE:
5582 case VMX_VMCS_GUEST_RSP:
5583 case VMX_VMCS_GUEST_RIP:
5584 case VMX_VMCS_GUEST_SYSENTER_ESP:
5585 case VMX_VMCS_GUEST_SYSENTER_EIP:
5586 {
5587 if (!(RT_HI_U32(u64Val)))
5588 {
5589 /* If this field is 64-bit, VT-x will zero out the top bits. */
5590 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5591 }
5592 else
5593 {
5594 /* Assert that only the 32->64 switcher case should ever come here. */
5595 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5596 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5597 }
5598 break;
5599 }
5600
5601 default:
5602 {
5603 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5604 rc = VERR_INVALID_PARAMETER;
5605 break;
5606 }
5607 }
5608 AssertRCReturn(rc, rc);
5609 return rc;
5610}
5611
5612
5613/**
5614 * Queue up a VMWRITE by using the VMCS write cache.
5615 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5616 *
5617 * @param pVCpu The cross context virtual CPU structure.
5618 * @param idxField The VMCS field encoding.
5619 * @param u64Val 16, 32 or 64-bit value.
5620 */
5621VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5622{
5623 AssertPtr(pVCpu);
5624 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5625
5626 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5627 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5628
5629 /* Make sure there are no duplicates. */
5630 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5631 {
5632 if (pCache->Write.aField[i] == idxField)
5633 {
5634 pCache->Write.aFieldVal[i] = u64Val;
5635 return VINF_SUCCESS;
5636 }
5637 }
5638
5639 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5640 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5641 pCache->Write.cValidEntries++;
5642 return VINF_SUCCESS;
5643}
5644#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5645
5646
5647/**
5648 * Sets up the usage of TSC-offsetting and updates the VMCS.
5649 *
5650 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5651 * VMX preemption timer.
5652 *
5653 * @returns VBox status code.
5654 * @param pVCpu The cross context virtual CPU structure.
5655 *
5656 * @remarks No-long-jump zone!!!
5657 */
5658static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5659{
5660 bool fOffsettedTsc;
5661 bool fParavirtTsc;
5662 PVM pVM = pVCpu->CTX_SUFF(pVM);
5663 uint64_t uTscOffset;
5664 if (pVM->hm.s.vmx.fUsePreemptTimer)
5665 {
5666 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5667
5668 /* Make sure the returned values have sane upper and lower boundaries. */
5669 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5670 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5671 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5672 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5673
5674 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5675 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5676 AssertRC(rc);
5677 }
5678 else
5679 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5680
5681 /** @todo later optimize this to be done elsewhere and not before every
5682 * VM-entry. */
5683 if (fParavirtTsc)
5684 {
5685 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5686 information before every VM-entry, hence disable it for performance sake. */
5687#if 0
5688 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5689 AssertRC(rc);
5690#endif
5691 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5692 }
5693
5694 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5695 if ( fOffsettedTsc
5696 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5697 {
5698 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5699 {
5700 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5701 AssertRC(rc);
5702 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5703 }
5704
5705 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5706 {
5707 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5708 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5709 AssertRC(rc);
5710 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5711 }
5712 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5713 }
5714 else
5715 {
5716 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5717 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5718 {
5719 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5720 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5721 AssertRC(rc);
5722 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5723 }
5724 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5725 }
5726}
5727
5728
5729/**
5730 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5731 * VM-exit interruption info type.
5732 *
5733 * @returns The IEM exception flags.
5734 * @param uVector The event vector.
5735 * @param uVmxVectorType The VMX event type.
5736 *
5737 * @remarks This function currently only constructs flags required for
5738 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5739 * and CR2 aspects of an exception are not included).
5740 */
5741static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5742{
5743 uint32_t fIemXcptFlags;
5744 switch (uVmxVectorType)
5745 {
5746 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5747 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5748 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5749 break;
5750
5751 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5752 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5753 break;
5754
5755 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5756 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5757 break;
5758
5759 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5760 {
5761 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5762 if (uVector == X86_XCPT_BP)
5763 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5764 else if (uVector == X86_XCPT_OF)
5765 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5766 else
5767 {
5768 fIemXcptFlags = 0;
5769 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5770 }
5771 break;
5772 }
5773
5774 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5775 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5776 break;
5777
5778 default:
5779 fIemXcptFlags = 0;
5780 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5781 break;
5782 }
5783 return fIemXcptFlags;
5784}
5785
5786
5787/**
5788 * Sets an event as a pending event to be injected into the guest.
5789 *
5790 * @param pVCpu The cross context virtual CPU structure.
5791 * @param u32IntInfo The VM-entry interruption-information field.
5792 * @param cbInstr The VM-entry instruction length in bytes (for software
5793 * interrupts, exceptions and privileged software
5794 * exceptions).
5795 * @param u32ErrCode The VM-entry exception error code.
5796 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5797 * page-fault.
5798 *
5799 * @remarks Statistics counter assumes this is a guest event being injected or
5800 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5801 * always incremented.
5802 */
5803DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5804 RTGCUINTPTR GCPtrFaultAddress)
5805{
5806 Assert(!pVCpu->hm.s.Event.fPending);
5807 pVCpu->hm.s.Event.fPending = true;
5808 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5809 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5810 pVCpu->hm.s.Event.cbInstr = cbInstr;
5811 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5812}
5813
5814
5815/**
5816 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5817 *
5818 * @param pVCpu The cross context virtual CPU structure.
5819 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5820 * out-of-sync. Make sure to update the required fields
5821 * before using them.
5822 */
5823DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5824{
5825 NOREF(pMixedCtx);
5826 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5827 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5828 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5829 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5830}
5831
5832
5833/**
5834 * Handle a condition that occurred while delivering an event through the guest
5835 * IDT.
5836 *
5837 * @returns Strict VBox status code (i.e. informational status codes too).
5838 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5839 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5840 * to continue execution of the guest which will delivery the \#DF.
5841 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5842 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5843 *
5844 * @param pVCpu The cross context virtual CPU structure.
5845 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5846 * out-of-sync. Make sure to update the required fields
5847 * before using them.
5848 * @param pVmxTransient Pointer to the VMX transient structure.
5849 *
5850 * @remarks No-long-jump zone!!!
5851 */
5852static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5853{
5854 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5855
5856 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5857 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5858 AssertRCReturn(rc2, rc2);
5859
5860 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5861 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5862 {
5863 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5864 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5865
5866 /*
5867 * If the event was a software interrupt (generated with INT n) or a software exception
5868 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5869 * can handle the VM-exit and continue guest execution which will re-execute the
5870 * instruction rather than re-injecting the exception, as that can cause premature
5871 * trips to ring-3 before injection and involve TRPM which currently has no way of
5872 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5873 * the problem).
5874 */
5875 IEMXCPTRAISE enmRaise;
5876 IEMXCPTRAISEINFO fRaiseInfo;
5877 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5878 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5879 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5880 {
5881 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5882 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5883 }
5884 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5885 {
5886 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5887 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5888 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5889 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5890 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5891 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5892 uExitVectorType), VERR_VMX_IPE_5);
5893
5894 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5895
5896 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5897 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5898 {
5899 pVmxTransient->fVectoringPF = true;
5900 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5901 }
5902 }
5903 else
5904 {
5905 /*
5906 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5907 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5908 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5909 */
5910 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5911 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5912 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5913 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5914 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5915 }
5916
5917 /*
5918 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5919 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5920 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5921 * subsequent VM-entry would fail.
5922 *
5923 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5924 */
5925 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5926 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5927 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5928 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5929 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5930 {
5931 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5932 }
5933
5934 switch (enmRaise)
5935 {
5936 case IEMXCPTRAISE_CURRENT_XCPT:
5937 {
5938 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5939 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5940 Assert(rcStrict == VINF_SUCCESS);
5941 break;
5942 }
5943
5944 case IEMXCPTRAISE_PREV_EVENT:
5945 {
5946 uint32_t u32ErrCode;
5947 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5948 {
5949 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5950 AssertRCReturn(rc2, rc2);
5951 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5952 }
5953 else
5954 u32ErrCode = 0;
5955
5956 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5957 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5958 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5959 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5960
5961 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5962 pVCpu->hm.s.Event.u32ErrCode));
5963 Assert(rcStrict == VINF_SUCCESS);
5964 break;
5965 }
5966
5967 case IEMXCPTRAISE_REEXEC_INSTR:
5968 Assert(rcStrict == VINF_SUCCESS);
5969 break;
5970
5971 case IEMXCPTRAISE_DOUBLE_FAULT:
5972 {
5973 /*
5974 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5975 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5976 */
5977 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5978 {
5979 pVmxTransient->fVectoringDoublePF = true;
5980 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5981 pMixedCtx->cr2));
5982 rcStrict = VINF_SUCCESS;
5983 }
5984 else
5985 {
5986 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5987 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5988 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5989 uIdtVector, uExitVector));
5990 rcStrict = VINF_HM_DOUBLE_FAULT;
5991 }
5992 break;
5993 }
5994
5995 case IEMXCPTRAISE_TRIPLE_FAULT:
5996 {
5997 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5998 rcStrict = VINF_EM_RESET;
5999 break;
6000 }
6001
6002 case IEMXCPTRAISE_CPU_HANG:
6003 {
6004 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
6005 rcStrict = VERR_EM_GUEST_CPU_HANG;
6006 break;
6007 }
6008
6009 default:
6010 {
6011 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
6012 rcStrict = VERR_VMX_IPE_2;
6013 break;
6014 }
6015 }
6016 }
6017 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
6018 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
6019 && uExitVector != X86_XCPT_DF
6020 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
6021 {
6022 /*
6023 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
6024 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
6025 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
6026 */
6027 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6028 {
6029 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
6030 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
6031 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6032 }
6033 }
6034
6035 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6036 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6037 return rcStrict;
6038}
6039
6040
6041/**
6042 * Imports a guest segment register from the current VMCS into
6043 * the guest-CPU context.
6044 *
6045 * @returns VBox status code.
6046 * @param pVCpu The cross context virtual CPU structure.
6047 * @param idxSel Index of the selector in the VMCS.
6048 * @param idxLimit Index of the segment limit in the VMCS.
6049 * @param idxBase Index of the segment base in the VMCS.
6050 * @param idxAccess Index of the access rights of the segment in the VMCS.
6051 * @param pSelReg Pointer to the segment selector.
6052 *
6053 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6054 * do not log!
6055 *
6056 * @remarks Never call this function directly!!! Use the
6057 * HMVMX_IMPORT_SREG() macro as that takes care
6058 * of whether to read from the VMCS cache or not.
6059 */
6060static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6061 PCPUMSELREG pSelReg)
6062{
6063 NOREF(pVCpu);
6064
6065 uint32_t u32Sel;
6066 uint32_t u32Limit;
6067 uint32_t u32Attr;
6068 uint64_t u64Base;
6069 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6070 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6071 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6072 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6073 AssertRCReturn(rc, rc);
6074
6075 pSelReg->Sel = (uint16_t)u32Sel;
6076 pSelReg->ValidSel = (uint16_t)u32Sel;
6077 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6078 pSelReg->u32Limit = u32Limit;
6079 pSelReg->u64Base = u64Base;
6080 pSelReg->Attr.u = u32Attr;
6081
6082 /*
6083 * If VT-x marks the segment as unusable, most other bits remain undefined:
6084 * - For CS the L, D and G bits have meaning.
6085 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6086 * - For the remaining data segments no bits are defined.
6087 *
6088 * The present bit and the unusable bit has been observed to be set at the
6089 * same time (the selector was supposed to be invalid as we started executing
6090 * a V8086 interrupt in ring-0).
6091 *
6092 * What should be important for the rest of the VBox code, is that the P bit is
6093 * cleared. Some of the other VBox code recognizes the unusable bit, but
6094 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6095 * safe side here, we'll strip off P and other bits we don't care about. If
6096 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6097 *
6098 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6099 */
6100 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6101 {
6102 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6103
6104 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6105 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6106 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6107#ifdef VBOX_STRICT
6108 VMMRZCallRing3Disable(pVCpu);
6109 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6110# ifdef DEBUG_bird
6111 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6112 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6113 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6114# endif
6115 VMMRZCallRing3Enable(pVCpu);
6116#endif
6117 }
6118 return VINF_SUCCESS;
6119}
6120
6121
6122/**
6123 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6124 *
6125 * @returns VBox status code.
6126 * @param pVCpu The cross context virtual CPU structure.
6127 *
6128 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6129 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6130 * instead!!!
6131 */
6132DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6133{
6134 uint64_t u64Val;
6135 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6136 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6137 {
6138 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6139 if (RT_SUCCESS(rc))
6140 {
6141 pCtx->rip = u64Val;
6142 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6143 }
6144 return rc;
6145 }
6146 return VINF_SUCCESS;
6147}
6148
6149
6150/**
6151 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6152 *
6153 * @returns VBox status code.
6154 * @param pVCpu The cross context virtual CPU structure.
6155 *
6156 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6157 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6158 * instead!!!
6159 */
6160DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6161{
6162 uint32_t u32Val;
6163 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6164 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6165 {
6166 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6167 if (RT_SUCCESS(rc))
6168 {
6169 pCtx->eflags.u32 = u32Val;
6170
6171 /* Restore eflags for real-on-v86-mode hack. */
6172 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6173 {
6174 pCtx->eflags.Bits.u1VM = 0;
6175 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6176 }
6177 }
6178 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6179 return rc;
6180 }
6181 return VINF_SUCCESS;
6182}
6183
6184
6185/**
6186 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6187 * context.
6188 *
6189 * @returns VBox status code.
6190 * @param pVCpu The cross context virtual CPU structure.
6191 *
6192 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6193 * do not log!
6194 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6195 * instead!!!
6196 */
6197DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6198{
6199 uint32_t u32Val;
6200 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6201 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6202 if (RT_SUCCESS(rc))
6203 {
6204 /*
6205 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6206 * might need them in hmR0VmxEvaluatePendingEvent().
6207 */
6208 if (!u32Val)
6209 {
6210 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6211 {
6212 rc = hmR0VmxImportGuestRip(pVCpu);
6213 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6214 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6215 }
6216
6217 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6218 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6219 }
6220 else
6221 {
6222 rc = hmR0VmxImportGuestRip(pVCpu);
6223 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6224
6225 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6226 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6227 {
6228 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6229 }
6230 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6231 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6232
6233 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6234 {
6235 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6236 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6237 }
6238 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6239 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6240 }
6241 }
6242 return rc;
6243}
6244
6245
6246/**
6247 * Worker for VMXR0ImportStateOnDemand.
6248 *
6249 * @returns VBox status code.
6250 * @param pVCpu The cross context virtual CPU structure.
6251 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6252 */
6253static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6254{
6255#define VMXLOCAL_BREAK_RC(a_rc) \
6256 if (RT_FAILURE(a_rc)) \
6257 break
6258
6259 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6260
6261 int rc = VINF_SUCCESS;
6262 PVM pVM = pVCpu->CTX_SUFF(pVM);
6263 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6264 uint64_t u64Val;
6265 uint32_t u32Val;
6266
6267 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6268
6269 /*
6270 * We disable interrupts to make the updating of the state and in particular
6271 * the fExtrn modification atomic wrt to preemption hooks.
6272 */
6273 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6274
6275 fWhat &= pCtx->fExtrn;
6276 if (fWhat)
6277 {
6278 do
6279 {
6280 if (fWhat & CPUMCTX_EXTRN_RIP)
6281 {
6282 rc = hmR0VmxImportGuestRip(pVCpu);
6283 VMXLOCAL_BREAK_RC(rc);
6284 }
6285
6286 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6287 {
6288 rc = hmR0VmxImportGuestRFlags(pVCpu);
6289 VMXLOCAL_BREAK_RC(rc);
6290 }
6291
6292 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6293 {
6294 rc = hmR0VmxImportGuestIntrState(pVCpu);
6295 VMXLOCAL_BREAK_RC(rc);
6296 }
6297
6298 if (fWhat & CPUMCTX_EXTRN_RSP)
6299 {
6300 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6301 VMXLOCAL_BREAK_RC(rc);
6302 pCtx->rsp = u64Val;
6303 }
6304
6305 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6306 {
6307 if (fWhat & CPUMCTX_EXTRN_CS)
6308 {
6309 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6310 VMXLOCAL_BREAK_RC(rc);
6311 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6312 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6313 }
6314 if (fWhat & CPUMCTX_EXTRN_SS)
6315 {
6316 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6317 VMXLOCAL_BREAK_RC(rc);
6318 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6319 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6320 }
6321 if (fWhat & CPUMCTX_EXTRN_DS)
6322 {
6323 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6324 VMXLOCAL_BREAK_RC(rc);
6325 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6326 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6327 }
6328 if (fWhat & CPUMCTX_EXTRN_ES)
6329 {
6330 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6331 VMXLOCAL_BREAK_RC(rc);
6332 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6333 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6334 }
6335 if (fWhat & CPUMCTX_EXTRN_FS)
6336 {
6337 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6338 VMXLOCAL_BREAK_RC(rc);
6339 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6340 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6341 }
6342 if (fWhat & CPUMCTX_EXTRN_GS)
6343 {
6344 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6345 VMXLOCAL_BREAK_RC(rc);
6346 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6347 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6348 }
6349 }
6350
6351 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6352 {
6353 if (fWhat & CPUMCTX_EXTRN_LDTR)
6354 {
6355 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6356 VMXLOCAL_BREAK_RC(rc);
6357 }
6358
6359 if (fWhat & CPUMCTX_EXTRN_GDTR)
6360 {
6361 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6362 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6363 VMXLOCAL_BREAK_RC(rc);
6364 pCtx->gdtr.pGdt = u64Val;
6365 pCtx->gdtr.cbGdt = u32Val;
6366 }
6367
6368 /* Guest IDTR. */
6369 if (fWhat & CPUMCTX_EXTRN_IDTR)
6370 {
6371 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6372 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6373 VMXLOCAL_BREAK_RC(rc);
6374 pCtx->idtr.pIdt = u64Val;
6375 pCtx->idtr.cbIdt = u32Val;
6376 }
6377
6378 /* Guest TR. */
6379 if (fWhat & CPUMCTX_EXTRN_TR)
6380 {
6381 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6382 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6383 {
6384 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6385 VMXLOCAL_BREAK_RC(rc);
6386 }
6387 }
6388 }
6389
6390 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6391 {
6392 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6393 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6394 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6395 pCtx->SysEnter.cs = u32Val;
6396 VMXLOCAL_BREAK_RC(rc);
6397 }
6398
6399#if HC_ARCH_BITS == 64
6400 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6401 {
6402 if ( pVM->hm.s.fAllow64BitGuests
6403 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6404 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6405 }
6406
6407 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6408 {
6409 if ( pVM->hm.s.fAllow64BitGuests
6410 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6411 {
6412 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6413 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6414 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6415 }
6416 }
6417#endif
6418
6419 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6420#if HC_ARCH_BITS == 32
6421 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6422#endif
6423 )
6424 {
6425 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6426 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6427 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6428 {
6429 switch (pMsr->u32Msr)
6430 {
6431#if HC_ARCH_BITS == 32
6432 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6433 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6434 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6435 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6436#endif
6437 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6438 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6439 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6440 default:
6441 {
6442 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6443 ASMSetFlags(fEFlags);
6444 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6445 cMsrs));
6446 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6447 }
6448 }
6449 }
6450 }
6451
6452 if (fWhat & CPUMCTX_EXTRN_DR7)
6453 {
6454 if (!pVCpu->hm.s.fUsingHyperDR7)
6455 {
6456 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6457 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6458 VMXLOCAL_BREAK_RC(rc);
6459 pCtx->dr[7] = u32Val;
6460 }
6461 }
6462
6463 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6464 {
6465 uint32_t u32Shadow;
6466 if (fWhat & CPUMCTX_EXTRN_CR0)
6467 {
6468 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6469 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6470 VMXLOCAL_BREAK_RC(rc);
6471 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6472 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6473 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6474 CPUMSetGuestCR0(pVCpu, u32Val);
6475 VMMRZCallRing3Enable(pVCpu);
6476 }
6477
6478 if (fWhat & CPUMCTX_EXTRN_CR4)
6479 {
6480 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6481 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6482 VMXLOCAL_BREAK_RC(rc);
6483 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6484 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6485 CPUMSetGuestCR4(pVCpu, u32Val);
6486 }
6487
6488 if (fWhat & CPUMCTX_EXTRN_CR3)
6489 {
6490 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6491 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6492 || ( pVM->hm.s.fNestedPaging
6493 && CPUMIsGuestPagingEnabledEx(pCtx)))
6494 {
6495 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6496 if (pCtx->cr3 != u64Val)
6497 {
6498 CPUMSetGuestCR3(pVCpu, u64Val);
6499 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6500 }
6501
6502 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6503 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6504 if (CPUMIsGuestInPAEModeEx(pCtx))
6505 {
6506 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6507 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6508 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6509 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6510 VMXLOCAL_BREAK_RC(rc);
6511 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6512 }
6513 }
6514 }
6515 }
6516 } while (0);
6517
6518 if (RT_SUCCESS(rc))
6519 {
6520 /* Update fExtrn. */
6521 pCtx->fExtrn &= ~fWhat;
6522
6523 /* If everything has been imported, clear the HM keeper bit. */
6524 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6525 {
6526 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6527 Assert(!pCtx->fExtrn);
6528 }
6529 }
6530 }
6531 else
6532 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6533
6534 ASMSetFlags(fEFlags);
6535
6536 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6537
6538 /*
6539 * Honor any pending CR3 updates.
6540 *
6541 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6542 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6543 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6544 *
6545 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6546 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6547 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6548 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6549 *
6550 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6551 */
6552 if (VMMRZCallRing3IsEnabled(pVCpu))
6553 {
6554 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6555 {
6556 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6557 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6558 }
6559
6560 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6561 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6562
6563 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6564 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6565 }
6566
6567 return VINF_SUCCESS;
6568#undef VMXLOCAL_BREAK_RC
6569}
6570
6571
6572/**
6573 * Saves the guest state from the VMCS into the guest-CPU context.
6574 *
6575 * @returns VBox status code.
6576 * @param pVCpu The cross context virtual CPU structure.
6577 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6578 */
6579VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6580{
6581 return hmR0VmxImportGuestState(pVCpu, fWhat);
6582}
6583
6584
6585/**
6586 * Check per-VM and per-VCPU force flag actions that require us to go back to
6587 * ring-3 for one reason or another.
6588 *
6589 * @returns Strict VBox status code (i.e. informational status codes too)
6590 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6591 * ring-3.
6592 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6593 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6594 * interrupts)
6595 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6596 * all EMTs to be in ring-3.
6597 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6598 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6599 * to the EM loop.
6600 *
6601 * @param pVCpu The cross context virtual CPU structure.
6602 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6603 * out-of-sync. Make sure to update the required fields
6604 * before using them.
6605 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6606 */
6607static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6608{
6609 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6610
6611 /*
6612 * Anything pending? Should be more likely than not if we're doing a good job.
6613 */
6614 PVM pVM = pVCpu->CTX_SUFF(pVM);
6615 if ( !fStepping
6616 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6617 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6618 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6619 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6620 return VINF_SUCCESS;
6621
6622 /* Pending PGM C3 sync. */
6623 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6624 {
6625 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6626 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6627 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6628 if (rcStrict2 != VINF_SUCCESS)
6629 {
6630 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6631 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6632 return rcStrict2;
6633 }
6634 }
6635
6636 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6637 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6638 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6639 {
6640 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6641 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6642 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6643 return rc2;
6644 }
6645
6646 /* Pending VM request packets, such as hardware interrupts. */
6647 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6648 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6649 {
6650 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6651 return VINF_EM_PENDING_REQUEST;
6652 }
6653
6654 /* Pending PGM pool flushes. */
6655 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6656 {
6657 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6658 return VINF_PGM_POOL_FLUSH_PENDING;
6659 }
6660
6661 /* Pending DMA requests. */
6662 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6663 {
6664 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6665 return VINF_EM_RAW_TO_R3;
6666 }
6667
6668 return VINF_SUCCESS;
6669}
6670
6671
6672/**
6673 * Converts any TRPM trap into a pending HM event. This is typically used when
6674 * entering from ring-3 (not longjmp returns).
6675 *
6676 * @param pVCpu The cross context virtual CPU structure.
6677 */
6678static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6679{
6680 Assert(TRPMHasTrap(pVCpu));
6681 Assert(!pVCpu->hm.s.Event.fPending);
6682
6683 uint8_t uVector;
6684 TRPMEVENT enmTrpmEvent;
6685 RTGCUINT uErrCode;
6686 RTGCUINTPTR GCPtrFaultAddress;
6687 uint8_t cbInstr;
6688
6689 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6690 AssertRC(rc);
6691
6692 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6693 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6694 if (enmTrpmEvent == TRPM_TRAP)
6695 {
6696 switch (uVector)
6697 {
6698 case X86_XCPT_NMI:
6699 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6700 break;
6701
6702 case X86_XCPT_BP:
6703 case X86_XCPT_OF:
6704 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6705 break;
6706
6707 case X86_XCPT_PF:
6708 case X86_XCPT_DF:
6709 case X86_XCPT_TS:
6710 case X86_XCPT_NP:
6711 case X86_XCPT_SS:
6712 case X86_XCPT_GP:
6713 case X86_XCPT_AC:
6714 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6715 RT_FALL_THRU();
6716 default:
6717 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6718 break;
6719 }
6720 }
6721 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6722 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6723 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6724 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6725 else
6726 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6727
6728 rc = TRPMResetTrap(pVCpu);
6729 AssertRC(rc);
6730 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6731 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6732
6733 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6734}
6735
6736
6737/**
6738 * Converts the pending HM event into a TRPM trap.
6739 *
6740 * @param pVCpu The cross context virtual CPU structure.
6741 */
6742static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6743{
6744 Assert(pVCpu->hm.s.Event.fPending);
6745
6746 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6747 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6748 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6749 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6750
6751 /* If a trap was already pending, we did something wrong! */
6752 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6753
6754 TRPMEVENT enmTrapType;
6755 switch (uVectorType)
6756 {
6757 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6758 enmTrapType = TRPM_HARDWARE_INT;
6759 break;
6760
6761 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6762 enmTrapType = TRPM_SOFTWARE_INT;
6763 break;
6764
6765 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6766 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6767 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6768 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6769 enmTrapType = TRPM_TRAP;
6770 break;
6771
6772 default:
6773 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6774 enmTrapType = TRPM_32BIT_HACK;
6775 break;
6776 }
6777
6778 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6779
6780 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6781 AssertRC(rc);
6782
6783 if (fErrorCodeValid)
6784 TRPMSetErrorCode(pVCpu, uErrorCode);
6785
6786 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6787 && uVector == X86_XCPT_PF)
6788 {
6789 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6790 }
6791 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6792 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6793 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6794 {
6795 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6796 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6797 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6798 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6799 }
6800
6801 /* Clear any pending events from the VMCS. */
6802 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6803 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6804
6805 /* We're now done converting the pending event. */
6806 pVCpu->hm.s.Event.fPending = false;
6807}
6808
6809
6810/**
6811 * Does the necessary state syncing before returning to ring-3 for any reason
6812 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6813 *
6814 * @returns VBox status code.
6815 * @param pVCpu The cross context virtual CPU structure.
6816 * @param fImportState Whether to import the guest state from the VMCS back
6817 * to the guest-CPU context.
6818 *
6819 * @remarks No-long-jmp zone!!!
6820 */
6821static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6822{
6823 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6824 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6825
6826 RTCPUID idCpu = RTMpCpuId();
6827 Log4Func(("HostCpuId=%u\n", idCpu));
6828
6829 /*
6830 * !!! IMPORTANT !!!
6831 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6832 */
6833
6834 /* Save the guest state if necessary. */
6835 if (fImportState)
6836 {
6837 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6838 AssertRCReturn(rc, rc);
6839 }
6840
6841 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6842 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6843 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6844
6845 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6846#ifdef VBOX_STRICT
6847 if (CPUMIsHyperDebugStateActive(pVCpu))
6848 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6849#endif
6850 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6851 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6852 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6853
6854#if HC_ARCH_BITS == 64
6855 /* Restore host-state bits that VT-x only restores partially. */
6856 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6857 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6858 {
6859 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6860 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6861 }
6862 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6863#endif
6864
6865 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6866 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6867 {
6868 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6869 if (!fImportState)
6870 {
6871 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6872 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6873 AssertRCReturn(rc, rc);
6874 }
6875 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6876 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6877 }
6878 else
6879 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6880
6881 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6882 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6883
6884 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6885 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6886 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6887 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6888 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6889 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6890 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6891 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6892 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6893
6894 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6895
6896 /** @todo This partially defeats the purpose of having preemption hooks.
6897 * The problem is, deregistering the hooks should be moved to a place that
6898 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6899 * context.
6900 */
6901 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6902 {
6903 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6904 AssertRCReturn(rc, rc);
6905
6906 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6907 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6908 }
6909 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6910 NOREF(idCpu);
6911
6912 return VINF_SUCCESS;
6913}
6914
6915
6916/**
6917 * Leaves the VT-x session.
6918 *
6919 * @returns VBox status code.
6920 * @param pVCpu The cross context virtual CPU structure.
6921 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6922 * out-of-sync. Make sure to update the required fields
6923 * before using them.
6924 *
6925 * @remarks No-long-jmp zone!!!
6926 */
6927static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6928{
6929 HM_DISABLE_PREEMPT();
6930 HMVMX_ASSERT_CPU_SAFE();
6931 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6932 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6933
6934 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6935 and done this from the VMXR0ThreadCtxCallback(). */
6936 if (!pVCpu->hm.s.fLeaveDone)
6937 {
6938 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6939 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6940 pVCpu->hm.s.fLeaveDone = true;
6941 }
6942 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6943
6944 /*
6945 * !!! IMPORTANT !!!
6946 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6947 */
6948
6949 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6950 /** @todo Deregistering here means we need to VMCLEAR always
6951 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6952 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6953 VMMR0ThreadCtxHookDisable(pVCpu);
6954
6955 /* Leave HM context. This takes care of local init (term). */
6956 int rc = HMR0LeaveCpu(pVCpu);
6957
6958 HM_RESTORE_PREEMPT();
6959 return rc;
6960}
6961
6962
6963/**
6964 * Does the necessary state syncing before doing a longjmp to ring-3.
6965 *
6966 * @returns VBox status code.
6967 * @param pVCpu The cross context virtual CPU structure.
6968 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6969 * out-of-sync. Make sure to update the required fields
6970 * before using them.
6971 *
6972 * @remarks No-long-jmp zone!!!
6973 */
6974DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6975{
6976 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6977}
6978
6979
6980/**
6981 * Take necessary actions before going back to ring-3.
6982 *
6983 * An action requires us to go back to ring-3. This function does the necessary
6984 * steps before we can safely return to ring-3. This is not the same as longjmps
6985 * to ring-3, this is voluntary and prepares the guest so it may continue
6986 * executing outside HM (recompiler/IEM).
6987 *
6988 * @returns VBox status code.
6989 * @param pVCpu The cross context virtual CPU structure.
6990 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6991 * out-of-sync. Make sure to update the required fields
6992 * before using them.
6993 * @param rcExit The reason for exiting to ring-3. Can be
6994 * VINF_VMM_UNKNOWN_RING3_CALL.
6995 */
6996static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6997{
6998 Assert(pVCpu);
6999 Assert(pMixedCtx);
7000 HMVMX_ASSERT_PREEMPT_SAFE();
7001
7002 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
7003 {
7004 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
7005 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
7006 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
7007 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
7008 }
7009
7010 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
7011 VMMRZCallRing3Disable(pVCpu);
7012 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
7013
7014 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
7015 if (pVCpu->hm.s.Event.fPending)
7016 {
7017 hmR0VmxPendingEventToTrpmTrap(pVCpu);
7018 Assert(!pVCpu->hm.s.Event.fPending);
7019 }
7020
7021 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
7022 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
7023
7024 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
7025 and if we're injecting an event we should have a TRPM trap pending. */
7026 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7027#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
7028 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
7029#endif
7030
7031 /* Save guest state and restore host state bits. */
7032 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7033 AssertRCReturn(rc, rc);
7034 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7035 /* Thread-context hooks are unregistered at this point!!! */
7036
7037 /* Sync recompiler state. */
7038 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7039 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7040 | CPUM_CHANGED_LDTR
7041 | CPUM_CHANGED_GDTR
7042 | CPUM_CHANGED_IDTR
7043 | CPUM_CHANGED_TR
7044 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7045 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7046 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7047 {
7048 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7049 }
7050
7051 Assert(!pVCpu->hm.s.fClearTrapFlag);
7052
7053 /* Update the exit-to-ring 3 reason. */
7054 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7055
7056 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7057 if (rcExit != VINF_EM_RAW_INTERRUPT)
7058 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7059
7060 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7061
7062 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7063 VMMRZCallRing3RemoveNotification(pVCpu);
7064 VMMRZCallRing3Enable(pVCpu);
7065
7066 return rc;
7067}
7068
7069
7070/**
7071 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7072 * longjump to ring-3 and possibly get preempted.
7073 *
7074 * @returns VBox status code.
7075 * @param pVCpu The cross context virtual CPU structure.
7076 * @param enmOperation The operation causing the ring-3 longjump.
7077 * @param pvUser Opaque pointer to the guest-CPU context. The data
7078 * may be out-of-sync. Make sure to update the required
7079 * fields before using them.
7080 */
7081static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7082{
7083 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7084 {
7085 /*
7086 * !!! IMPORTANT !!!
7087 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7088 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7089 */
7090 VMMRZCallRing3RemoveNotification(pVCpu);
7091 VMMRZCallRing3Disable(pVCpu);
7092 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7093 RTThreadPreemptDisable(&PreemptState);
7094
7095 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7096 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7097
7098#if HC_ARCH_BITS == 64
7099 /* Restore host-state bits that VT-x only restores partially. */
7100 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7101 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7102 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7103 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7104#endif
7105
7106 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7107 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7108 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7109
7110 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7111 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7112 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7113 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7114 {
7115 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7116 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7117 }
7118
7119 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7120 VMMR0ThreadCtxHookDisable(pVCpu);
7121 HMR0LeaveCpu(pVCpu);
7122 RTThreadPreemptRestore(&PreemptState);
7123 return VINF_SUCCESS;
7124 }
7125
7126 Assert(pVCpu);
7127 Assert(pvUser);
7128 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7129 HMVMX_ASSERT_PREEMPT_SAFE();
7130
7131 VMMRZCallRing3Disable(pVCpu);
7132 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7133
7134 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7135
7136 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7137 AssertRCReturn(rc, rc);
7138
7139 VMMRZCallRing3Enable(pVCpu);
7140 return VINF_SUCCESS;
7141}
7142
7143
7144/**
7145 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7146 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7147 *
7148 * @param pVCpu The cross context virtual CPU structure.
7149 */
7150DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7151{
7152 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7153 {
7154 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7155 {
7156 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7157 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7158 AssertRC(rc);
7159 Log4Func(("Setup interrupt-window exiting\n"));
7160 }
7161 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7162}
7163
7164
7165/**
7166 * Clears the interrupt-window exiting control in the VMCS.
7167 *
7168 * @param pVCpu The cross context virtual CPU structure.
7169 */
7170DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7171{
7172 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7173 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7174 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7175 AssertRC(rc);
7176 Log4Func(("Cleared interrupt-window exiting\n"));
7177}
7178
7179
7180/**
7181 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7182 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7183 *
7184 * @param pVCpu The cross context virtual CPU structure.
7185 */
7186DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7187{
7188 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7189 {
7190 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7191 {
7192 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7193 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7194 AssertRC(rc);
7195 Log4Func(("Setup NMI-window exiting\n"));
7196 }
7197 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7198}
7199
7200
7201/**
7202 * Clears the NMI-window exiting control in the VMCS.
7203 *
7204 * @param pVCpu The cross context virtual CPU structure.
7205 */
7206DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7207{
7208 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7209 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7210 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7211 AssertRC(rc);
7212 Log4Func(("Cleared NMI-window exiting\n"));
7213}
7214
7215
7216/**
7217 * Evaluates the event to be delivered to the guest and sets it as the pending
7218 * event.
7219 *
7220 * @returns The VT-x guest-interruptibility state.
7221 * @param pVCpu The cross context virtual CPU structure.
7222 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7223 * out-of-sync. Make sure to update the required fields
7224 * before using them.
7225 */
7226static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7227{
7228 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7229 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7230 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7231 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7232 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7233
7234 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7235 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7236 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7237 Assert(!TRPMHasTrap(pVCpu));
7238
7239 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7240 APICUpdatePendingInterrupts(pVCpu);
7241
7242 /*
7243 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7244 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7245 */
7246 /** @todo SMI. SMIs take priority over NMIs. */
7247 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7248 {
7249 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7250 if ( !pVCpu->hm.s.Event.fPending
7251 && !fBlockNmi
7252 && !fBlockSti
7253 && !fBlockMovSS)
7254 {
7255 Log4Func(("Pending NMI\n"));
7256 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7257 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7258
7259 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7260 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7261 }
7262 else
7263 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7264 }
7265 /*
7266 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7267 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7268 */
7269 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7270 && !pVCpu->hm.s.fSingleInstruction)
7271 {
7272 Assert(!DBGFIsStepping(pVCpu));
7273 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7274 AssertRCReturn(rc, 0);
7275 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7276 if ( !pVCpu->hm.s.Event.fPending
7277 && !fBlockInt
7278 && !fBlockSti
7279 && !fBlockMovSS)
7280 {
7281 uint8_t u8Interrupt;
7282 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7283 if (RT_SUCCESS(rc))
7284 {
7285 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7286 uint32_t u32IntInfo = u8Interrupt
7287 | VMX_EXIT_INTERRUPTION_INFO_VALID
7288 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7289
7290 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7291 }
7292 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7293 {
7294 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7295 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7296 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7297
7298 /*
7299 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7300 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7301 * need to re-set this force-flag here.
7302 */
7303 }
7304 else
7305 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7306 }
7307 else
7308 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7309 }
7310
7311 return fIntrState;
7312}
7313
7314
7315/**
7316 * Sets a pending-debug exception to be delivered to the guest if the guest is
7317 * single-stepping in the VMCS.
7318 *
7319 * @param pVCpu The cross context virtual CPU structure.
7320 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7321 * out-of-sync. Make sure to update the required fields
7322 * before using them.
7323 */
7324DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7325{
7326 RT_NOREF(pVCpu);
7327 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7328 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7329}
7330
7331
7332/**
7333 * Injects any pending events into the guest if the guest is in a state to
7334 * receive them.
7335 *
7336 * @returns Strict VBox status code (i.e. informational status codes too).
7337 * @param pVCpu The cross context virtual CPU structure.
7338 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7339 * out-of-sync. Make sure to update the required fields
7340 * before using them.
7341 * @param fIntrState The VT-x guest-interruptibility state.
7342 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7343 * return VINF_EM_DBG_STEPPED if the event was
7344 * dispatched directly.
7345 */
7346static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7347{
7348 HMVMX_ASSERT_PREEMPT_SAFE();
7349 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7350
7351 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7352 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7353
7354 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7355 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7356 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7357 Assert(!TRPMHasTrap(pVCpu));
7358
7359 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7360 if (pVCpu->hm.s.Event.fPending)
7361 {
7362 /*
7363 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7364 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7365 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7366 *
7367 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7368 */
7369 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7370#ifdef VBOX_STRICT
7371 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7372 {
7373 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7374 Assert(!fBlockInt);
7375 Assert(!fBlockSti);
7376 Assert(!fBlockMovSS);
7377 }
7378 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7379 {
7380 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7381 Assert(!fBlockSti);
7382 Assert(!fBlockMovSS);
7383 Assert(!fBlockNmi);
7384 }
7385#endif
7386 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7387 uIntType));
7388 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7389 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7390 &fIntrState);
7391 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7392
7393 /* Update the interruptibility-state as it could have been changed by
7394 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7395 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7396 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7397
7398 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7399 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7400 else
7401 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7402 }
7403
7404 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7405 if ( fBlockSti
7406 || fBlockMovSS)
7407 {
7408 if (!pVCpu->hm.s.fSingleInstruction)
7409 {
7410 /*
7411 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7412 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7413 * See Intel spec. 27.3.4 "Saving Non-Register State".
7414 */
7415 Assert(!DBGFIsStepping(pVCpu));
7416 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7417 AssertRCReturn(rc, rc);
7418 if (pMixedCtx->eflags.Bits.u1TF)
7419 {
7420 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7421 AssertRCReturn(rc2, rc2);
7422 }
7423 }
7424 else if (pMixedCtx->eflags.Bits.u1TF)
7425 {
7426 /*
7427 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7428 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7429 */
7430 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7431 fIntrState = 0;
7432 }
7433 }
7434
7435 /*
7436 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7437 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7438 */
7439 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7440 AssertRCReturn(rc3, rc3);
7441
7442 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7443 NOREF(fBlockMovSS); NOREF(fBlockSti);
7444 return rcStrict;
7445}
7446
7447
7448/**
7449 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7450 *
7451 * @param pVCpu The cross context virtual CPU structure.
7452 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7453 * out-of-sync. Make sure to update the required fields
7454 * before using them.
7455 */
7456DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7457{
7458 NOREF(pMixedCtx);
7459 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7460 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7461}
7462
7463
7464/**
7465 * Injects a double-fault (\#DF) exception into the VM.
7466 *
7467 * @returns Strict VBox status code (i.e. informational status codes too).
7468 * @param pVCpu The cross context virtual CPU structure.
7469 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7470 * out-of-sync. Make sure to update the required fields
7471 * before using them.
7472 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7473 * and should return VINF_EM_DBG_STEPPED if the event
7474 * is injected directly (register modified by us, not
7475 * by hardware on VM-entry).
7476 * @param pfIntrState Pointer to the current guest interruptibility-state.
7477 * This interruptibility-state will be updated if
7478 * necessary. This cannot not be NULL.
7479 */
7480DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7481{
7482 NOREF(pMixedCtx);
7483 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7484 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7485 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7486 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7487 pfIntrState);
7488}
7489
7490
7491/**
7492 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7493 *
7494 * @param pVCpu The cross context virtual CPU structure.
7495 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7496 * out-of-sync. Make sure to update the required fields
7497 * before using them.
7498 */
7499DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7500{
7501 NOREF(pMixedCtx);
7502 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7503 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7504 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7505}
7506
7507
7508/**
7509 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7510 *
7511 * @param pVCpu The cross context virtual CPU structure.
7512 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7513 * out-of-sync. Make sure to update the required fields
7514 * before using them.
7515 * @param cbInstr The value of RIP that is to be pushed on the guest
7516 * stack.
7517 */
7518DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7519{
7520 NOREF(pMixedCtx);
7521 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7522 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7523 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7524}
7525
7526
7527/**
7528 * Injects a general-protection (\#GP) fault into the VM.
7529 *
7530 * @returns Strict VBox status code (i.e. informational status codes too).
7531 * @param pVCpu The cross context virtual CPU structure.
7532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7533 * out-of-sync. Make sure to update the required fields
7534 * before using them.
7535 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7536 * mode, i.e. in real-mode it's not valid).
7537 * @param u32ErrorCode The error code associated with the \#GP.
7538 * @param fStepping Whether we're running in
7539 * hmR0VmxRunGuestCodeStep() and should return
7540 * VINF_EM_DBG_STEPPED if the event is injected
7541 * directly (register modified by us, not by
7542 * hardware on VM-entry).
7543 * @param pfIntrState Pointer to the current guest interruptibility-state.
7544 * This interruptibility-state will be updated if
7545 * necessary. This cannot not be NULL.
7546 */
7547DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7548 bool fStepping, uint32_t *pfIntrState)
7549{
7550 NOREF(pMixedCtx);
7551 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7552 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7553 if (fErrorCodeValid)
7554 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7555 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7556 pfIntrState);
7557}
7558
7559
7560#if 0 /* unused */
7561/**
7562 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7563 * VM.
7564 *
7565 * @param pVCpu The cross context virtual CPU structure.
7566 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7567 * out-of-sync. Make sure to update the required fields
7568 * before using them.
7569 * @param u32ErrorCode The error code associated with the \#GP.
7570 */
7571DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7572{
7573 NOREF(pMixedCtx);
7574 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7575 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7576 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7577 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7578}
7579#endif /* unused */
7580
7581
7582/**
7583 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7584 *
7585 * @param pVCpu The cross context virtual CPU structure.
7586 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7587 * out-of-sync. Make sure to update the required fields
7588 * before using them.
7589 * @param uVector The software interrupt vector number.
7590 * @param cbInstr The value of RIP that is to be pushed on the guest
7591 * stack.
7592 */
7593DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7594{
7595 NOREF(pMixedCtx);
7596 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7597 if ( uVector == X86_XCPT_BP
7598 || uVector == X86_XCPT_OF)
7599 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7600 else
7601 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7602 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7603}
7604
7605
7606/**
7607 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7608 * stack.
7609 *
7610 * @returns Strict VBox status code (i.e. informational status codes too).
7611 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7612 * @param pVM The cross context VM structure.
7613 * @param pMixedCtx Pointer to the guest-CPU context.
7614 * @param uValue The value to push to the guest stack.
7615 */
7616DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7617{
7618 /*
7619 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7620 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7621 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7622 */
7623 if (pMixedCtx->sp == 1)
7624 return VINF_EM_RESET;
7625 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7626 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7627 AssertRC(rc);
7628 return rc;
7629}
7630
7631
7632/**
7633 * Injects an event into the guest upon VM-entry by updating the relevant fields
7634 * in the VM-entry area in the VMCS.
7635 *
7636 * @returns Strict VBox status code (i.e. informational status codes too).
7637 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7638 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7639 *
7640 * @param pVCpu The cross context virtual CPU structure.
7641 * @param u64IntInfo The VM-entry interruption-information field.
7642 * @param cbInstr The VM-entry instruction length in bytes (for
7643 * software interrupts, exceptions and privileged
7644 * software exceptions).
7645 * @param u32ErrCode The VM-entry exception error code.
7646 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7647 * @param pfIntrState Pointer to the current guest interruptibility-state.
7648 * This interruptibility-state will be updated if
7649 * necessary. This cannot not be NULL.
7650 * @param fStepping Whether we're running in
7651 * hmR0VmxRunGuestCodeStep() and should return
7652 * VINF_EM_DBG_STEPPED if the event is injected
7653 * directly (register modified by us, not by
7654 * hardware on VM-entry).
7655 */
7656static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7657 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7658{
7659 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7660 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7661 Assert(pfIntrState);
7662
7663 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7664 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7665 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7666 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7667
7668#ifdef VBOX_STRICT
7669 /*
7670 * Validate the error-code-valid bit for hardware exceptions.
7671 * No error codes for exceptions in real-mode.
7672 *
7673 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7674 */
7675 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7676 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7677 {
7678 switch (uVector)
7679 {
7680 case X86_XCPT_PF:
7681 case X86_XCPT_DF:
7682 case X86_XCPT_TS:
7683 case X86_XCPT_NP:
7684 case X86_XCPT_SS:
7685 case X86_XCPT_GP:
7686 case X86_XCPT_AC:
7687 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7688 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7689 RT_FALL_THRU();
7690 default:
7691 break;
7692 }
7693 }
7694#endif
7695
7696 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7697 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7698 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7699
7700 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7701
7702 /*
7703 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7704 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7705 * interrupt handler in the (real-mode) guest.
7706 *
7707 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7708 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7709 */
7710 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7711 {
7712 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7713 {
7714 /*
7715 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7716 * set the deliver-error-code bit.
7717 *
7718 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7719 */
7720 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7721 }
7722 else
7723 {
7724 PVM pVM = pVCpu->CTX_SUFF(pVM);
7725 Assert(PDMVmmDevHeapIsEnabled(pVM));
7726 Assert(pVM->hm.s.vmx.pRealModeTSS);
7727
7728 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7729 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7730 | CPUMCTX_EXTRN_TABLE_MASK
7731 | CPUMCTX_EXTRN_RIP
7732 | CPUMCTX_EXTRN_RSP
7733 | CPUMCTX_EXTRN_RFLAGS);
7734 AssertRCReturn(rc2, rc2);
7735
7736 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7737 size_t const cbIdtEntry = sizeof(X86IDTR16);
7738 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7739 {
7740 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7741 if (uVector == X86_XCPT_DF)
7742 return VINF_EM_RESET;
7743
7744 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7745 if (uVector == X86_XCPT_GP)
7746 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7747
7748 /*
7749 * If we're injecting an event with no valid IDT entry, inject a #GP.
7750 * No error codes for exceptions in real-mode.
7751 *
7752 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7753 */
7754 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7755 pfIntrState);
7756 }
7757
7758 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7759 uint16_t uGuestIp = pMixedCtx->ip;
7760 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7761 {
7762 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7763 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7764 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7765 }
7766 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7767 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7768
7769 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7770 X86IDTR16 IdtEntry;
7771 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7772 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7773 AssertRCReturn(rc2, rc2);
7774
7775 /* Construct the stack frame for the interrupt/exception handler. */
7776 VBOXSTRICTRC rcStrict;
7777 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7778 if (rcStrict == VINF_SUCCESS)
7779 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7780 if (rcStrict == VINF_SUCCESS)
7781 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7782
7783 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7784 if (rcStrict == VINF_SUCCESS)
7785 {
7786 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7787 pMixedCtx->rip = IdtEntry.offSel;
7788 pMixedCtx->cs.Sel = IdtEntry.uSel;
7789 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7790 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7791 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7792 && uVector == X86_XCPT_PF)
7793 pMixedCtx->cr2 = GCPtrFaultAddress;
7794
7795 /* If any other guest-state bits are changed here, make sure to update
7796 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7797 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7798 | HM_CHANGED_GUEST_CR2
7799 | HM_CHANGED_GUEST_RIP
7800 | HM_CHANGED_GUEST_RFLAGS
7801 | HM_CHANGED_GUEST_RSP);
7802
7803 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7804 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7805 {
7806 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7807 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7808 Log4Func(("Clearing inhibition due to STI\n"));
7809 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7810 }
7811 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7812 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7813
7814 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7815 it, if we are returning to ring-3 before executing guest code. */
7816 pVCpu->hm.s.Event.fPending = false;
7817
7818 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7819 if (fStepping)
7820 rcStrict = VINF_EM_DBG_STEPPED;
7821 }
7822 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7823 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7824 return rcStrict;
7825 }
7826 }
7827
7828 /* Validate. */
7829 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7830 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7831 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7832
7833 /* Inject. */
7834 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7835 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7836 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7837 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7838 AssertRCReturn(rc, rc);
7839
7840 /* Update CR2. */
7841 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7842 && uVector == X86_XCPT_PF)
7843 pMixedCtx->cr2 = GCPtrFaultAddress;
7844
7845 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7846
7847 return VINF_SUCCESS;
7848}
7849
7850
7851/**
7852 * Clears the interrupt-window exiting control in the VMCS and if necessary
7853 * clears the current event in the VMCS as well.
7854 *
7855 * @returns VBox status code.
7856 * @param pVCpu The cross context virtual CPU structure.
7857 *
7858 * @remarks Use this function only to clear events that have not yet been
7859 * delivered to the guest but are injected in the VMCS!
7860 * @remarks No-long-jump zone!!!
7861 */
7862static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7863{
7864 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7865 {
7866 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7867 Log4Func(("Cleared interrupt widow\n"));
7868 }
7869
7870 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7871 {
7872 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7873 Log4Func(("Cleared interrupt widow\n"));
7874 }
7875}
7876
7877
7878/**
7879 * Enters the VT-x session.
7880 *
7881 * @returns VBox status code.
7882 * @param pVCpu The cross context virtual CPU structure.
7883 * @param pHostCpu Pointer to the global CPU info struct.
7884 */
7885VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7886{
7887 AssertPtr(pVCpu);
7888 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7889 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7890 RT_NOREF(pHostCpu);
7891
7892 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7893 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7894 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7895
7896#ifdef VBOX_STRICT
7897 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7898 RTCCUINTREG uHostCR4 = ASMGetCR4();
7899 if (!(uHostCR4 & X86_CR4_VMXE))
7900 {
7901 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7902 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7903 }
7904#endif
7905
7906 /*
7907 * Load the VCPU's VMCS as the current (and active) one.
7908 */
7909 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7910 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7911 if (RT_FAILURE(rc))
7912 return rc;
7913
7914 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7915 pVCpu->hm.s.fLeaveDone = false;
7916 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7917
7918 return VINF_SUCCESS;
7919}
7920
7921
7922/**
7923 * The thread-context callback (only on platforms which support it).
7924 *
7925 * @param enmEvent The thread-context event.
7926 * @param pVCpu The cross context virtual CPU structure.
7927 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7928 * @thread EMT(pVCpu)
7929 */
7930VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7931{
7932 NOREF(fGlobalInit);
7933
7934 switch (enmEvent)
7935 {
7936 case RTTHREADCTXEVENT_OUT:
7937 {
7938 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7939 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7940 VMCPU_ASSERT_EMT(pVCpu);
7941
7942 /* No longjmps (logger flushes, locks) in this fragile context. */
7943 VMMRZCallRing3Disable(pVCpu);
7944 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7945
7946 /*
7947 * Restore host-state (FPU, debug etc.)
7948 */
7949 if (!pVCpu->hm.s.fLeaveDone)
7950 {
7951 /*
7952 * Do -not- import the guest-state here as we might already be in the middle of importing
7953 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7954 */
7955 hmR0VmxLeave(pVCpu, false /* fImportState */);
7956 pVCpu->hm.s.fLeaveDone = true;
7957 }
7958
7959 /* Leave HM context, takes care of local init (term). */
7960 int rc = HMR0LeaveCpu(pVCpu);
7961 AssertRC(rc); NOREF(rc);
7962
7963 /* Restore longjmp state. */
7964 VMMRZCallRing3Enable(pVCpu);
7965 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7966 break;
7967 }
7968
7969 case RTTHREADCTXEVENT_IN:
7970 {
7971 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7972 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7973 VMCPU_ASSERT_EMT(pVCpu);
7974
7975 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7976 VMMRZCallRing3Disable(pVCpu);
7977 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7978
7979 /* Initialize the bare minimum state required for HM. This takes care of
7980 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7981 int rc = hmR0EnterCpu(pVCpu);
7982 AssertRC(rc);
7983 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7984 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7985
7986 /* Load the active VMCS as the current one. */
7987 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7988 {
7989 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7990 AssertRC(rc); NOREF(rc);
7991 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7992 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7993 }
7994 pVCpu->hm.s.fLeaveDone = false;
7995
7996 /* Restore longjmp state. */
7997 VMMRZCallRing3Enable(pVCpu);
7998 break;
7999 }
8000
8001 default:
8002 break;
8003 }
8004}
8005
8006
8007/**
8008 * Exports the host state into the VMCS host-state area.
8009 * Sets up the VM-exit MSR-load area.
8010 *
8011 * The CPU state will be loaded from these fields on every successful VM-exit.
8012 *
8013 * @returns VBox status code.
8014 * @param pVCpu The cross context virtual CPU structure.
8015 *
8016 * @remarks No-long-jump zone!!!
8017 */
8018static int hmR0VmxExportHostState(PVMCPU pVCpu)
8019{
8020 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8021
8022 int rc = VINF_SUCCESS;
8023 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8024 {
8025 rc = hmR0VmxExportHostControlRegs();
8026 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8027
8028 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
8029 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8030
8031 rc = hmR0VmxExportHostMsrs(pVCpu);
8032 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8033
8034 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8035 }
8036 return rc;
8037}
8038
8039
8040/**
8041 * Saves the host state in the VMCS host-state.
8042 *
8043 * @returns VBox status code.
8044 * @param pVCpu The cross context virtual CPU structure.
8045 *
8046 * @remarks No-long-jump zone!!!
8047 */
8048VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8049{
8050 AssertPtr(pVCpu);
8051 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8052
8053 /*
8054 * Export the host state here while entering HM context.
8055 * When thread-context hooks are used, we might get preempted and have to re-save the host
8056 * state but most of the time we won't be, so do it here before we disable interrupts.
8057 */
8058 return hmR0VmxExportHostState(pVCpu);
8059}
8060
8061
8062/**
8063 * Exports the guest state into the VMCS guest-state area.
8064 *
8065 * The will typically be done before VM-entry when the guest-CPU state and the
8066 * VMCS state may potentially be out of sync.
8067 *
8068 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8069 * VM-entry controls.
8070 * Sets up the appropriate VMX non-root function to execute guest code based on
8071 * the guest CPU mode.
8072 *
8073 * @returns VBox strict status code.
8074 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8075 * without unrestricted guest access and the VMMDev is not presently
8076 * mapped (e.g. EFI32).
8077 *
8078 * @param pVCpu The cross context virtual CPU structure.
8079 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8080 * out-of-sync. Make sure to update the required fields
8081 * before using them.
8082 *
8083 * @remarks No-long-jump zone!!!
8084 */
8085static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8086{
8087 AssertPtr(pVCpu);
8088 AssertPtr(pMixedCtx);
8089 HMVMX_ASSERT_PREEMPT_SAFE();
8090
8091 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8092
8093 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8094
8095 /* Determine real-on-v86 mode. */
8096 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8097 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8098 && CPUMIsGuestInRealModeEx(pMixedCtx))
8099 {
8100 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8101 }
8102
8103 /*
8104 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8105 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8106 */
8107 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8108 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8109
8110 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8111 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8112 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8113
8114 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8115 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8116 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8117
8118 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8119 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8120
8121 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8122 if (rcStrict == VINF_SUCCESS)
8123 { /* likely */ }
8124 else
8125 {
8126 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8127 return rcStrict;
8128 }
8129
8130 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8131 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8132
8133 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8134 may alter controls if we determine we don't have to swap EFER after all. */
8135 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8136 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8137
8138 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8139 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8140
8141 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8142 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8143 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8144
8145 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8146 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8147 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8148 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8149 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8150 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8151
8152 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8153 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8154 | HM_CHANGED_GUEST_CR2
8155 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8156 | HM_CHANGED_GUEST_X87
8157 | HM_CHANGED_GUEST_SSE_AVX
8158 | HM_CHANGED_GUEST_OTHER_XSAVE
8159 | HM_CHANGED_GUEST_XCRx
8160 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8161 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8162 | HM_CHANGED_GUEST_TSC_AUX
8163 | HM_CHANGED_GUEST_OTHER_MSRS
8164 | HM_CHANGED_GUEST_HWVIRT
8165 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8166
8167 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8168 return rc;
8169}
8170
8171
8172/**
8173 * Exports the state shared between the host and guest into the VMCS.
8174 *
8175 * @param pVCpu The cross context virtual CPU structure.
8176 * @param pCtx Pointer to the guest-CPU context.
8177 *
8178 * @remarks No-long-jump zone!!!
8179 */
8180static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8181{
8182 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8183 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8184
8185 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8186 {
8187 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8188 AssertRC(rc);
8189 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8190
8191 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8192 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8193 {
8194 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8195 AssertRC(rc);
8196 }
8197 }
8198
8199 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8200 {
8201 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8202 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8203 }
8204
8205 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8206 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8207}
8208
8209
8210/**
8211 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8212 *
8213 * @returns Strict VBox status code (i.e. informational status codes too).
8214 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8215 * without unrestricted guest access and the VMMDev is not presently
8216 * mapped (e.g. EFI32).
8217 *
8218 * @param pVCpu The cross context virtual CPU structure.
8219 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8220 * out-of-sync. Make sure to update the required fields
8221 * before using them.
8222 *
8223 * @remarks No-long-jump zone!!!
8224 */
8225static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8226{
8227 HMVMX_ASSERT_PREEMPT_SAFE();
8228 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8229 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8230
8231#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8232 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8233#endif
8234
8235 /*
8236 * For many exits it's only RIP that changes and hence try to export it first
8237 * without going through a lot of change flag checks.
8238 */
8239 VBOXSTRICTRC rcStrict;
8240 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8241 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8242 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8243 {
8244 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8245 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8246 { /* likely */}
8247 else
8248 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8249 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8250 }
8251 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8252 {
8253 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8254 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8255 { /* likely */}
8256 else
8257 {
8258 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8259 VBOXSTRICTRC_VAL(rcStrict)));
8260 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8261 return rcStrict;
8262 }
8263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8264 }
8265 else
8266 rcStrict = VINF_SUCCESS;
8267
8268#ifdef VBOX_STRICT
8269 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8270 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8271 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8272 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8273 ("fCtxChanged=%#RX64\n", fCtxChanged));
8274#endif
8275 return rcStrict;
8276}
8277
8278
8279/**
8280 * Does the preparations before executing guest code in VT-x.
8281 *
8282 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8283 * recompiler/IEM. We must be cautious what we do here regarding committing
8284 * guest-state information into the VMCS assuming we assuredly execute the
8285 * guest in VT-x mode.
8286 *
8287 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8288 * the common-state (TRPM/forceflags), we must undo those changes so that the
8289 * recompiler/IEM can (and should) use them when it resumes guest execution.
8290 * Otherwise such operations must be done when we can no longer exit to ring-3.
8291 *
8292 * @returns Strict VBox status code (i.e. informational status codes too).
8293 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8294 * have been disabled.
8295 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8296 * double-fault into the guest.
8297 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8298 * dispatched directly.
8299 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8300 *
8301 * @param pVCpu The cross context virtual CPU structure.
8302 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8303 * out-of-sync. Make sure to update the required fields
8304 * before using them.
8305 * @param pVmxTransient Pointer to the VMX transient structure.
8306 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8307 * us ignore some of the reasons for returning to
8308 * ring-3, and return VINF_EM_DBG_STEPPED if event
8309 * dispatching took place.
8310 */
8311static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8312{
8313 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8314
8315#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8316 PGMRZDynMapFlushAutoSet(pVCpu);
8317#endif
8318
8319 /* Check force flag actions that might require us to go back to ring-3. */
8320 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8321 if (rcStrict == VINF_SUCCESS)
8322 { /* FFs doesn't get set all the time. */ }
8323 else
8324 return rcStrict;
8325
8326 /*
8327 * Setup the virtualized-APIC accesses.
8328 *
8329 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8330 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8331 *
8332 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8333 */
8334 PVM pVM = pVCpu->CTX_SUFF(pVM);
8335 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8336 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8337 && PDMHasApic(pVM))
8338 {
8339 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8340 Assert(u64MsrApicBase);
8341 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8342
8343 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8344
8345 /* Unalias any existing mapping. */
8346 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8347 AssertRCReturn(rc, rc);
8348
8349 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8350 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8351 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8352 AssertRCReturn(rc, rc);
8353
8354 /* Update the per-VCPU cache of the APIC base MSR. */
8355 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8356 }
8357
8358 if (TRPMHasTrap(pVCpu))
8359 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8360 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8361
8362 /*
8363 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8364 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8365 * also result in triple-faulting the VM.
8366 */
8367 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8368 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8369 { /* likely */ }
8370 else
8371 {
8372 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8373 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8374 return rcStrict;
8375 }
8376
8377 /*
8378 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8379 * import CR3 themselves. We will need to update them here as even as late as the above
8380 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8381 * the below force flags to be set.
8382 */
8383 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8384 {
8385 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8386 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8387 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8388 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8389 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8390 }
8391 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8392 {
8393 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8394 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8395 }
8396
8397 /*
8398 * No longjmps to ring-3 from this point on!!!
8399 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8400 * This also disables flushing of the R0-logger instance (if any).
8401 */
8402 VMMRZCallRing3Disable(pVCpu);
8403
8404 /*
8405 * Export the guest state bits.
8406 *
8407 * We cannot perform longjmps while loading the guest state because we do not preserve the
8408 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8409 * CPU migration.
8410 *
8411 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8412 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8413 * Hence, loading of the guest state needs to be done -after- injection of events.
8414 */
8415 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8416 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8417 { /* likely */ }
8418 else
8419 {
8420 VMMRZCallRing3Enable(pVCpu);
8421 return rcStrict;
8422 }
8423
8424 /*
8425 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8426 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8427 * preemption disabled for a while. Since this is purly to aid the
8428 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8429 * disable interrupt on NT.
8430 *
8431 * We need to check for force-flags that could've possible been altered since we last
8432 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8433 * see @bugref{6398}).
8434 *
8435 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8436 * to ring-3 before executing guest code.
8437 */
8438 pVmxTransient->fEFlags = ASMIntDisableFlags();
8439
8440 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8441 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8442 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8443 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8444 {
8445 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8446 {
8447 pVCpu->hm.s.Event.fPending = false;
8448
8449 /*
8450 * We've injected any pending events. This is really the point of no return (to ring-3).
8451 *
8452 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8453 * returns from this function, so don't enable them here.
8454 */
8455 return VINF_SUCCESS;
8456 }
8457
8458 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8459 rcStrict = VINF_EM_RAW_INTERRUPT;
8460 }
8461 else
8462 {
8463 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8464 rcStrict = VINF_EM_RAW_TO_R3;
8465 }
8466
8467 ASMSetFlags(pVmxTransient->fEFlags);
8468 VMMRZCallRing3Enable(pVCpu);
8469
8470 return rcStrict;
8471}
8472
8473
8474/**
8475 * Prepares to run guest code in VT-x and we've committed to doing so. This
8476 * means there is no backing out to ring-3 or anywhere else at this
8477 * point.
8478 *
8479 * @param pVCpu The cross context virtual CPU structure.
8480 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8481 * out-of-sync. Make sure to update the required fields
8482 * before using them.
8483 * @param pVmxTransient Pointer to the VMX transient structure.
8484 *
8485 * @remarks Called with preemption disabled.
8486 * @remarks No-long-jump zone!!!
8487 */
8488static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8489{
8490 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8491 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8492 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8493
8494 /*
8495 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8496 */
8497 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8498 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8499
8500 PVM pVM = pVCpu->CTX_SUFF(pVM);
8501 if (!CPUMIsGuestFPUStateActive(pVCpu))
8502 {
8503 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8504 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8505 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8506 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8507 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8508 }
8509
8510 /*
8511 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8512 */
8513 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8514 && pVCpu->hm.s.vmx.cMsrs > 0)
8515 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8516
8517 /*
8518 * Re-save the host state bits as we may've been preempted (only happens when
8519 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8520 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8521 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8522 * See @bugref{8432}.
8523 */
8524 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8525 {
8526 int rc = hmR0VmxExportHostState(pVCpu);
8527 AssertRC(rc);
8528 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8529 }
8530 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8531
8532 /*
8533 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8534 */
8535 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8536 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8537 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8538
8539 /* Store status of the shared guest-host state at the time of VM-entry. */
8540#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8541 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8542 {
8543 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8544 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8545 }
8546 else
8547#endif
8548 {
8549 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8550 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8551 }
8552
8553 /*
8554 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8555 */
8556 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8557 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8558
8559 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8560 RTCPUID idCurrentCpu = pCpu->idCpu;
8561 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8562 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8563 {
8564 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8565 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8566 }
8567
8568 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8569 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8570 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8571 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8572
8573 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8574
8575 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8576 to start executing. */
8577
8578 /*
8579 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8580 */
8581 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8582 {
8583 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8584 {
8585 bool fMsrUpdated;
8586 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8587 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8588 &fMsrUpdated);
8589 AssertRC(rc2);
8590 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8591 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8592 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8593 }
8594 else
8595 {
8596 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8597 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8598 }
8599 }
8600
8601 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8602 {
8603 bool fMsrUpdated;
8604 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8605 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8606 &fMsrUpdated);
8607 AssertRC(rc2);
8608 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8609 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8610 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8611 }
8612
8613#ifdef VBOX_STRICT
8614 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8615 hmR0VmxCheckHostEferMsr(pVCpu);
8616 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8617#endif
8618#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8619 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8620 {
8621 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8622 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8623 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8624 }
8625#endif
8626}
8627
8628
8629/**
8630 * Performs some essential restoration of state after running guest code in
8631 * VT-x.
8632 *
8633 * @param pVCpu The cross context virtual CPU structure.
8634 * @param pVmxTransient Pointer to the VMX transient structure.
8635 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8636 *
8637 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8638 *
8639 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8640 * unconditionally when it is safe to do so.
8641 */
8642static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8643{
8644 uint64_t const uHostTsc = ASMReadTSC();
8645 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8646
8647 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8648 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8649 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8650 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8651 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8652 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8653
8654 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8655 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8656
8657 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8658 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8659 Assert(!ASMIntAreEnabled());
8660 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8661
8662#if HC_ARCH_BITS == 64
8663 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8664#endif
8665#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8666 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8667 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8668 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8669#else
8670 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8671#endif
8672#ifdef VBOX_STRICT
8673 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8674#endif
8675 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8676
8677 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8678 uint32_t uExitReason;
8679 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8680 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8681 AssertRC(rc);
8682 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8683 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8684
8685 if (rcVMRun == VINF_SUCCESS)
8686 {
8687 /*
8688 * Update the VM-exit history array here even if the VM-entry failed due to:
8689 * - Invalid guest state.
8690 * - MSR loading.
8691 * - Machine-check event.
8692 *
8693 * In any of the above cases we will still have a "valid" VM-exit reason
8694 * despite @a fVMEntryFailed being false.
8695 *
8696 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8697 *
8698 * Note! We don't have CS or RIP at this point. Will probably address that later
8699 * by amending the history entry added here.
8700 */
8701 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8702 UINT64_MAX, uHostTsc);
8703
8704 if (!pVmxTransient->fVMEntryFailed)
8705 {
8706 VMMRZCallRing3Enable(pVCpu);
8707
8708 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8709 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8710
8711#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8712 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8713 AssertRC(rc);
8714#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8715 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8716 AssertRC(rc);
8717#else
8718 /*
8719 * Import the guest-interruptibility state always as we need it while evaluating
8720 * injecting events on re-entry.
8721 *
8722 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8723 * checking for real-mode while exporting the state because all bits that cause
8724 * mode changes wrt CR0 are intercepted.
8725 */
8726 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8727 AssertRC(rc);
8728#endif
8729
8730 /*
8731 * Sync the TPR shadow with our APIC state.
8732 */
8733 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8734 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8735 {
8736 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8737 AssertRC(rc);
8738 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8739 }
8740
8741 return;
8742 }
8743 }
8744 else
8745 {
8746 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8747 }
8748
8749 VMMRZCallRing3Enable(pVCpu);
8750}
8751
8752
8753/**
8754 * Runs the guest code using VT-x the normal way.
8755 *
8756 * @returns VBox status code.
8757 * @param pVCpu The cross context virtual CPU structure.
8758 * @param pCtx Pointer to the guest-CPU context.
8759 *
8760 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8761 */
8762static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8763{
8764 VMXTRANSIENT VmxTransient;
8765 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8766 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8767 uint32_t cLoops = 0;
8768
8769 for (;; cLoops++)
8770 {
8771 Assert(!HMR0SuspendPending());
8772 HMVMX_ASSERT_CPU_SAFE();
8773
8774 /* Preparatory work for running guest code, this may force us to return
8775 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8776 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8777 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8778 if (rcStrict != VINF_SUCCESS)
8779 break;
8780
8781 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8782 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8783 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8784
8785 /* Restore any residual host-state and save any bits shared between host
8786 and guest into the guest-CPU state. Re-enables interrupts! */
8787 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8788
8789 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8790 if (RT_SUCCESS(rcRun))
8791 { /* very likely */ }
8792 else
8793 {
8794 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8795 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8796 return rcRun;
8797 }
8798
8799 /* Profile the VM-exit. */
8800 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8801 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8802 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8803 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8804 HMVMX_START_EXIT_DISPATCH_PROF();
8805
8806 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8807
8808 /* Handle the VM-exit. */
8809#ifdef HMVMX_USE_FUNCTION_TABLE
8810 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8811#else
8812 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8813#endif
8814 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8815 if (rcStrict == VINF_SUCCESS)
8816 {
8817 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8818 continue; /* likely */
8819 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8820 rcStrict = VINF_EM_RAW_INTERRUPT;
8821 }
8822 break;
8823 }
8824
8825 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8826 return rcStrict;
8827}
8828
8829
8830
8831/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8832 * probes.
8833 *
8834 * The following few functions and associated structure contains the bloat
8835 * necessary for providing detailed debug events and dtrace probes as well as
8836 * reliable host side single stepping. This works on the principle of
8837 * "subclassing" the normal execution loop and workers. We replace the loop
8838 * method completely and override selected helpers to add necessary adjustments
8839 * to their core operation.
8840 *
8841 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8842 * any performance for debug and analysis features.
8843 *
8844 * @{
8845 */
8846
8847/**
8848 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8849 * the debug run loop.
8850 */
8851typedef struct VMXRUNDBGSTATE
8852{
8853 /** The RIP we started executing at. This is for detecting that we stepped. */
8854 uint64_t uRipStart;
8855 /** The CS we started executing with. */
8856 uint16_t uCsStart;
8857
8858 /** Whether we've actually modified the 1st execution control field. */
8859 bool fModifiedProcCtls : 1;
8860 /** Whether we've actually modified the 2nd execution control field. */
8861 bool fModifiedProcCtls2 : 1;
8862 /** Whether we've actually modified the exception bitmap. */
8863 bool fModifiedXcptBitmap : 1;
8864
8865 /** We desire the modified the CR0 mask to be cleared. */
8866 bool fClearCr0Mask : 1;
8867 /** We desire the modified the CR4 mask to be cleared. */
8868 bool fClearCr4Mask : 1;
8869 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8870 uint32_t fCpe1Extra;
8871 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8872 uint32_t fCpe1Unwanted;
8873 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8874 uint32_t fCpe2Extra;
8875 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8876 uint32_t bmXcptExtra;
8877 /** The sequence number of the Dtrace provider settings the state was
8878 * configured against. */
8879 uint32_t uDtraceSettingsSeqNo;
8880 /** VM-exits to check (one bit per VM-exit). */
8881 uint32_t bmExitsToCheck[3];
8882
8883 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8884 uint32_t fProcCtlsInitial;
8885 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8886 uint32_t fProcCtls2Initial;
8887 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8888 uint32_t bmXcptInitial;
8889} VMXRUNDBGSTATE;
8890AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8891typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8892
8893
8894/**
8895 * Initializes the VMXRUNDBGSTATE structure.
8896 *
8897 * @param pVCpu The cross context virtual CPU structure of the
8898 * calling EMT.
8899 * @param pCtx The CPU register context to go with @a pVCpu.
8900 * @param pDbgState The structure to initialize.
8901 */
8902static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8903{
8904 pDbgState->uRipStart = pCtx->rip;
8905 pDbgState->uCsStart = pCtx->cs.Sel;
8906
8907 pDbgState->fModifiedProcCtls = false;
8908 pDbgState->fModifiedProcCtls2 = false;
8909 pDbgState->fModifiedXcptBitmap = false;
8910 pDbgState->fClearCr0Mask = false;
8911 pDbgState->fClearCr4Mask = false;
8912 pDbgState->fCpe1Extra = 0;
8913 pDbgState->fCpe1Unwanted = 0;
8914 pDbgState->fCpe2Extra = 0;
8915 pDbgState->bmXcptExtra = 0;
8916 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8917 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8918 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8919}
8920
8921
8922/**
8923 * Updates the VMSC fields with changes requested by @a pDbgState.
8924 *
8925 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8926 * immediately before executing guest code, i.e. when interrupts are disabled.
8927 * We don't check status codes here as we cannot easily assert or return in the
8928 * latter case.
8929 *
8930 * @param pVCpu The cross context virtual CPU structure.
8931 * @param pDbgState The debug state.
8932 */
8933static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8934{
8935 /*
8936 * Ensure desired flags in VMCS control fields are set.
8937 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8938 *
8939 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8940 * there should be no stale data in pCtx at this point.
8941 */
8942 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8943 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8944 {
8945 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8946 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8947 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8948 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8949 pDbgState->fModifiedProcCtls = true;
8950 }
8951
8952 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8953 {
8954 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8955 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8956 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8957 pDbgState->fModifiedProcCtls2 = true;
8958 }
8959
8960 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8961 {
8962 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8963 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8964 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8965 pDbgState->fModifiedXcptBitmap = true;
8966 }
8967
8968 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8969 {
8970 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8971 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8972 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8973 }
8974
8975 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8976 {
8977 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8978 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8979 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8980 }
8981}
8982
8983
8984static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8985{
8986 /*
8987 * Restore VM-exit control settings as we may not reenter this function the
8988 * next time around.
8989 */
8990 /* We reload the initial value, trigger what we can of recalculations the
8991 next time around. From the looks of things, that's all that's required atm. */
8992 if (pDbgState->fModifiedProcCtls)
8993 {
8994 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8995 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8996 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8997 AssertRCReturn(rc2, rc2);
8998 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8999 }
9000
9001 /* We're currently the only ones messing with this one, so just restore the
9002 cached value and reload the field. */
9003 if ( pDbgState->fModifiedProcCtls2
9004 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
9005 {
9006 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
9007 AssertRCReturn(rc2, rc2);
9008 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
9009 }
9010
9011 /* If we've modified the exception bitmap, we restore it and trigger
9012 reloading and partial recalculation the next time around. */
9013 if (pDbgState->fModifiedXcptBitmap)
9014 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
9015
9016 return rcStrict;
9017}
9018
9019
9020/**
9021 * Configures VM-exit controls for current DBGF and DTrace settings.
9022 *
9023 * This updates @a pDbgState and the VMCS execution control fields to reflect
9024 * the necessary VM-exits demanded by DBGF and DTrace.
9025 *
9026 * @param pVCpu The cross context virtual CPU structure.
9027 * @param pDbgState The debug state.
9028 * @param pVmxTransient Pointer to the VMX transient structure. May update
9029 * fUpdateTscOffsettingAndPreemptTimer.
9030 */
9031static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
9032{
9033 /*
9034 * Take down the dtrace serial number so we can spot changes.
9035 */
9036 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9037 ASMCompilerBarrier();
9038
9039 /*
9040 * We'll rebuild most of the middle block of data members (holding the
9041 * current settings) as we go along here, so start by clearing it all.
9042 */
9043 pDbgState->bmXcptExtra = 0;
9044 pDbgState->fCpe1Extra = 0;
9045 pDbgState->fCpe1Unwanted = 0;
9046 pDbgState->fCpe2Extra = 0;
9047 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9048 pDbgState->bmExitsToCheck[i] = 0;
9049
9050 /*
9051 * Software interrupts (INT XXh) - no idea how to trigger these...
9052 */
9053 PVM pVM = pVCpu->CTX_SUFF(pVM);
9054 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9055 || VBOXVMM_INT_SOFTWARE_ENABLED())
9056 {
9057 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9058 }
9059
9060 /*
9061 * INT3 breakpoints - triggered by #BP exceptions.
9062 */
9063 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9064 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9065
9066 /*
9067 * Exception bitmap and XCPT events+probes.
9068 */
9069 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9070 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9071 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9072
9073 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9074 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9075 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9076 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9077 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9078 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9079 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9080 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9081 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9082 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9083 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9084 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9085 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9086 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9087 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9088 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9089 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9090 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9091
9092 if (pDbgState->bmXcptExtra)
9093 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9094
9095 /*
9096 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9097 *
9098 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9099 * So, when adding/changing/removing please don't forget to update it.
9100 *
9101 * Some of the macros are picking up local variables to save horizontal space,
9102 * (being able to see it in a table is the lesser evil here).
9103 */
9104#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9105 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9106 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9107#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9108 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9109 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9110 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9111 } else do { } while (0)
9112#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9113 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9114 { \
9115 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9116 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9117 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9118 } else do { } while (0)
9119#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9120 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9121 { \
9122 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9123 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9124 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9125 } else do { } while (0)
9126#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9127 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9128 { \
9129 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9130 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9131 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9132 } else do { } while (0)
9133
9134 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9135 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9136 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9137 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9138 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9139
9140 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9141 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9142 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9143 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9144 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9146 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9147 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9148 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9150 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9152 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9154 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9156 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9157 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9158 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9160 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9161 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9162 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9163 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9164 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9165 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9166 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9167 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9168 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9170 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9172 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9173 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9174 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9175 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9176
9177 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9178 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9179 {
9180 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9181 | CPUMCTX_EXTRN_CR4
9182 | CPUMCTX_EXTRN_APIC_TPR);
9183 AssertRC(rc);
9184
9185#if 0 /** @todo fix me */
9186 pDbgState->fClearCr0Mask = true;
9187 pDbgState->fClearCr4Mask = true;
9188#endif
9189 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9190 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9191 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9192 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9193 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9194 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9195 require clearing here and in the loop if we start using it. */
9196 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9197 }
9198 else
9199 {
9200 if (pDbgState->fClearCr0Mask)
9201 {
9202 pDbgState->fClearCr0Mask = false;
9203 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9204 }
9205 if (pDbgState->fClearCr4Mask)
9206 {
9207 pDbgState->fClearCr4Mask = false;
9208 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9209 }
9210 }
9211 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9212 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9213
9214 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9215 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9216 {
9217 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9218 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9219 }
9220 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9221 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9222
9223 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9224 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9225 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9226 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9227 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9228 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9229 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9230 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9231#if 0 /** @todo too slow, fix handler. */
9232 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9233#endif
9234 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9235
9236 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9237 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9238 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9239 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9240 {
9241 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9242 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9243 }
9244 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9245 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9246 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9247 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9248
9249 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9250 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9251 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9252 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9253 {
9254 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9255 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9256 }
9257 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9258 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9259 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9260 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9261
9262 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9263 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9264 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9265 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9266 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9267 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9268 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9269 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9270 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9271 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9272 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9273 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9274 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9275 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9276 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9277 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9278 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9279 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9280 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9281 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9282 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9283 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9284
9285#undef IS_EITHER_ENABLED
9286#undef SET_ONLY_XBM_IF_EITHER_EN
9287#undef SET_CPE1_XBM_IF_EITHER_EN
9288#undef SET_CPEU_XBM_IF_EITHER_EN
9289#undef SET_CPE2_XBM_IF_EITHER_EN
9290
9291 /*
9292 * Sanitize the control stuff.
9293 */
9294 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9295 if (pDbgState->fCpe2Extra)
9296 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9297 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9298 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9299 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9300 {
9301 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9302 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9303 }
9304
9305 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9306 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9307 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9308 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9309}
9310
9311
9312/**
9313 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9314 * appropriate.
9315 *
9316 * The caller has checked the VM-exit against the
9317 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9318 * already, so we don't have to do that either.
9319 *
9320 * @returns Strict VBox status code (i.e. informational status codes too).
9321 * @param pVCpu The cross context virtual CPU structure.
9322 * @param pMixedCtx Pointer to the guest-CPU context.
9323 * @param pVmxTransient Pointer to the VMX-transient structure.
9324 * @param uExitReason The VM-exit reason.
9325 *
9326 * @remarks The name of this function is displayed by dtrace, so keep it short
9327 * and to the point. No longer than 33 chars long, please.
9328 */
9329static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9330 uint32_t uExitReason)
9331{
9332 /*
9333 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9334 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9335 *
9336 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9337 * does. Must add/change/remove both places. Same ordering, please.
9338 *
9339 * Added/removed events must also be reflected in the next section
9340 * where we dispatch dtrace events.
9341 */
9342 bool fDtrace1 = false;
9343 bool fDtrace2 = false;
9344 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9345 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9346 uint32_t uEventArg = 0;
9347#define SET_EXIT(a_EventSubName) \
9348 do { \
9349 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9350 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9351 } while (0)
9352#define SET_BOTH(a_EventSubName) \
9353 do { \
9354 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9355 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9356 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9357 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9358 } while (0)
9359 switch (uExitReason)
9360 {
9361 case VMX_EXIT_MTF:
9362 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9363
9364 case VMX_EXIT_XCPT_OR_NMI:
9365 {
9366 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9367 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9368 {
9369 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9370 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9371 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9372 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9373 {
9374 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9375 {
9376 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9377 uEventArg = pVmxTransient->uExitIntErrorCode;
9378 }
9379 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9380 switch (enmEvent1)
9381 {
9382 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9383 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9384 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9385 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9386 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9387 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9388 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9389 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9390 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9391 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9392 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9393 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9394 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9395 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9396 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9397 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9398 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9399 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9400 default: break;
9401 }
9402 }
9403 else
9404 AssertFailed();
9405 break;
9406
9407 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9408 uEventArg = idxVector;
9409 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9410 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9411 break;
9412 }
9413 break;
9414 }
9415
9416 case VMX_EXIT_TRIPLE_FAULT:
9417 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9418 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9419 break;
9420 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9421 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9422 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9423 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9424 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9425
9426 /* Instruction specific VM-exits: */
9427 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9428 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9429 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9430 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9431 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9432 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9433 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9434 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9435 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9436 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9437 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9438 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9439 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9440 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9441 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9442 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9443 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9444 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9445 case VMX_EXIT_MOV_CRX:
9446 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9447 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9448 SET_BOTH(CRX_READ);
9449 else
9450 SET_BOTH(CRX_WRITE);
9451 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9452 break;
9453 case VMX_EXIT_MOV_DRX:
9454 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9455 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9456 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9457 SET_BOTH(DRX_READ);
9458 else
9459 SET_BOTH(DRX_WRITE);
9460 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9461 break;
9462 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9463 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9464 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9465 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9466 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9467 case VMX_EXIT_XDTR_ACCESS:
9468 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9469 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9470 {
9471 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9472 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9473 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9474 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9475 }
9476 break;
9477
9478 case VMX_EXIT_TR_ACCESS:
9479 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9480 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9481 {
9482 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9483 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9484 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9485 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9486 }
9487 break;
9488
9489 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9490 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9491 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9492 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9493 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9494 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9495 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9496 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9497 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9498 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9499 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9500
9501 /* Events that aren't relevant at this point. */
9502 case VMX_EXIT_EXT_INT:
9503 case VMX_EXIT_INT_WINDOW:
9504 case VMX_EXIT_NMI_WINDOW:
9505 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9506 case VMX_EXIT_PREEMPT_TIMER:
9507 case VMX_EXIT_IO_INSTR:
9508 break;
9509
9510 /* Errors and unexpected events. */
9511 case VMX_EXIT_INIT_SIGNAL:
9512 case VMX_EXIT_SIPI:
9513 case VMX_EXIT_IO_SMI:
9514 case VMX_EXIT_SMI:
9515 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9516 case VMX_EXIT_ERR_MSR_LOAD:
9517 case VMX_EXIT_ERR_MACHINE_CHECK:
9518 break;
9519
9520 default:
9521 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9522 break;
9523 }
9524#undef SET_BOTH
9525#undef SET_EXIT
9526
9527 /*
9528 * Dtrace tracepoints go first. We do them here at once so we don't
9529 * have to copy the guest state saving and stuff a few dozen times.
9530 * Down side is that we've got to repeat the switch, though this time
9531 * we use enmEvent since the probes are a subset of what DBGF does.
9532 */
9533 if (fDtrace1 || fDtrace2)
9534 {
9535 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9536 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9537 switch (enmEvent1)
9538 {
9539 /** @todo consider which extra parameters would be helpful for each probe. */
9540 case DBGFEVENT_END: break;
9541 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9543 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9550 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9551 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9552 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9553 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9554 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9559 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9560 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9561 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9569 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9570 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9571 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9572 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9573 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9574 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9575 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9590 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9591 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9592 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9593 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9594 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9595 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9596 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9599 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9607 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9608 }
9609 switch (enmEvent2)
9610 {
9611 /** @todo consider which extra parameters would be helpful for each probe. */
9612 case DBGFEVENT_END: break;
9613 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9615 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9617 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9623 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9624 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9625 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9626 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9627 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9628 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9629 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9631 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9632 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9633 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9634 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9635 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9636 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9637 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9638 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9639 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9640 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9641 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9642 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9643 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9644 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9645 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9646 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9647 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9648 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9649 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9650 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9651 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9652 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9653 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9654 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9655 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9656 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9657 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9658 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9659 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9660 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9661 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9662 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9663 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9664 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9665 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9666 }
9667 }
9668
9669 /*
9670 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9671 * the DBGF call will do a full check).
9672 *
9673 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9674 * Note! If we have to events, we prioritize the first, i.e. the instruction
9675 * one, in order to avoid event nesting.
9676 */
9677 PVM pVM = pVCpu->CTX_SUFF(pVM);
9678 if ( enmEvent1 != DBGFEVENT_END
9679 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9680 {
9681 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9682 if (rcStrict != VINF_SUCCESS)
9683 return rcStrict;
9684 }
9685 else if ( enmEvent2 != DBGFEVENT_END
9686 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9687 {
9688 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9689 if (rcStrict != VINF_SUCCESS)
9690 return rcStrict;
9691 }
9692
9693 return VINF_SUCCESS;
9694}
9695
9696
9697/**
9698 * Single-stepping VM-exit filtering.
9699 *
9700 * This is preprocessing the VM-exits and deciding whether we've gotten far
9701 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9702 * handling is performed.
9703 *
9704 * @returns Strict VBox status code (i.e. informational status codes too).
9705 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9706 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9707 * out-of-sync. Make sure to update the required
9708 * fields before using them.
9709 * @param pVmxTransient Pointer to the VMX-transient structure.
9710 * @param uExitReason The VM-exit reason.
9711 * @param pDbgState The debug state.
9712 */
9713DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9714 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9715{
9716 /*
9717 * Expensive (saves context) generic dtrace VM-exit probe.
9718 */
9719 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9720 { /* more likely */ }
9721 else
9722 {
9723 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9724 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9725 AssertRC(rc);
9726 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9727 }
9728
9729 /*
9730 * Check for host NMI, just to get that out of the way.
9731 */
9732 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9733 { /* normally likely */ }
9734 else
9735 {
9736 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9737 AssertRCReturn(rc2, rc2);
9738 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9739 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9740 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9741 }
9742
9743 /*
9744 * Check for single stepping event if we're stepping.
9745 */
9746 if (pVCpu->hm.s.fSingleInstruction)
9747 {
9748 switch (uExitReason)
9749 {
9750 case VMX_EXIT_MTF:
9751 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9752
9753 /* Various events: */
9754 case VMX_EXIT_XCPT_OR_NMI:
9755 case VMX_EXIT_EXT_INT:
9756 case VMX_EXIT_TRIPLE_FAULT:
9757 case VMX_EXIT_INT_WINDOW:
9758 case VMX_EXIT_NMI_WINDOW:
9759 case VMX_EXIT_TASK_SWITCH:
9760 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9761 case VMX_EXIT_APIC_ACCESS:
9762 case VMX_EXIT_EPT_VIOLATION:
9763 case VMX_EXIT_EPT_MISCONFIG:
9764 case VMX_EXIT_PREEMPT_TIMER:
9765
9766 /* Instruction specific VM-exits: */
9767 case VMX_EXIT_CPUID:
9768 case VMX_EXIT_GETSEC:
9769 case VMX_EXIT_HLT:
9770 case VMX_EXIT_INVD:
9771 case VMX_EXIT_INVLPG:
9772 case VMX_EXIT_RDPMC:
9773 case VMX_EXIT_RDTSC:
9774 case VMX_EXIT_RSM:
9775 case VMX_EXIT_VMCALL:
9776 case VMX_EXIT_VMCLEAR:
9777 case VMX_EXIT_VMLAUNCH:
9778 case VMX_EXIT_VMPTRLD:
9779 case VMX_EXIT_VMPTRST:
9780 case VMX_EXIT_VMREAD:
9781 case VMX_EXIT_VMRESUME:
9782 case VMX_EXIT_VMWRITE:
9783 case VMX_EXIT_VMXOFF:
9784 case VMX_EXIT_VMXON:
9785 case VMX_EXIT_MOV_CRX:
9786 case VMX_EXIT_MOV_DRX:
9787 case VMX_EXIT_IO_INSTR:
9788 case VMX_EXIT_RDMSR:
9789 case VMX_EXIT_WRMSR:
9790 case VMX_EXIT_MWAIT:
9791 case VMX_EXIT_MONITOR:
9792 case VMX_EXIT_PAUSE:
9793 case VMX_EXIT_XDTR_ACCESS:
9794 case VMX_EXIT_TR_ACCESS:
9795 case VMX_EXIT_INVEPT:
9796 case VMX_EXIT_RDTSCP:
9797 case VMX_EXIT_INVVPID:
9798 case VMX_EXIT_WBINVD:
9799 case VMX_EXIT_XSETBV:
9800 case VMX_EXIT_RDRAND:
9801 case VMX_EXIT_INVPCID:
9802 case VMX_EXIT_VMFUNC:
9803 case VMX_EXIT_RDSEED:
9804 case VMX_EXIT_XSAVES:
9805 case VMX_EXIT_XRSTORS:
9806 {
9807 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9808 | CPUMCTX_EXTRN_CS);
9809 AssertRCReturn(rc, rc);
9810 if ( pMixedCtx->rip != pDbgState->uRipStart
9811 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9812 return VINF_EM_DBG_STEPPED;
9813 break;
9814 }
9815
9816 /* Errors and unexpected events: */
9817 case VMX_EXIT_INIT_SIGNAL:
9818 case VMX_EXIT_SIPI:
9819 case VMX_EXIT_IO_SMI:
9820 case VMX_EXIT_SMI:
9821 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9822 case VMX_EXIT_ERR_MSR_LOAD:
9823 case VMX_EXIT_ERR_MACHINE_CHECK:
9824 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9825 break;
9826
9827 default:
9828 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9829 break;
9830 }
9831 }
9832
9833 /*
9834 * Check for debugger event breakpoints and dtrace probes.
9835 */
9836 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9837 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9838 {
9839 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9840 if (rcStrict != VINF_SUCCESS)
9841 return rcStrict;
9842 }
9843
9844 /*
9845 * Normal processing.
9846 */
9847#ifdef HMVMX_USE_FUNCTION_TABLE
9848 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9849#else
9850 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9851#endif
9852}
9853
9854
9855/**
9856 * Single steps guest code using VT-x.
9857 *
9858 * @returns Strict VBox status code (i.e. informational status codes too).
9859 * @param pVCpu The cross context virtual CPU structure.
9860 * @param pCtx Pointer to the guest-CPU context.
9861 *
9862 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9863 */
9864static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9865{
9866 VMXTRANSIENT VmxTransient;
9867 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9868
9869 /* Set HMCPU indicators. */
9870 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9871 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9872 pVCpu->hm.s.fDebugWantRdTscExit = false;
9873 pVCpu->hm.s.fUsingDebugLoop = true;
9874
9875 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9876 VMXRUNDBGSTATE DbgState;
9877 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9878 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9879
9880 /*
9881 * The loop.
9882 */
9883 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9884 for (uint32_t cLoops = 0; ; cLoops++)
9885 {
9886 Assert(!HMR0SuspendPending());
9887 HMVMX_ASSERT_CPU_SAFE();
9888 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9889
9890 /*
9891 * Preparatory work for running guest code, this may force us to return
9892 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9893 */
9894 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9895 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9896 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9897 if (rcStrict != VINF_SUCCESS)
9898 break;
9899
9900 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9901 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9902
9903 /*
9904 * Now we can run the guest code.
9905 */
9906 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9907
9908 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9909
9910 /*
9911 * Restore any residual host-state and save any bits shared between host
9912 * and guest into the guest-CPU state. Re-enables interrupts!
9913 */
9914 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9915
9916 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9917 if (RT_SUCCESS(rcRun))
9918 { /* very likely */ }
9919 else
9920 {
9921 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9922 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9923 return rcRun;
9924 }
9925
9926 /* Profile the VM-exit. */
9927 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9928 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9929 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9930 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9931 HMVMX_START_EXIT_DISPATCH_PROF();
9932
9933 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9934
9935 /*
9936 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9937 */
9938 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9939 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9940 if (rcStrict != VINF_SUCCESS)
9941 break;
9942 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9943 {
9944 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9945 rcStrict = VINF_EM_RAW_INTERRUPT;
9946 break;
9947 }
9948
9949 /*
9950 * Stepping: Did the RIP change, if so, consider it a single step.
9951 * Otherwise, make sure one of the TFs gets set.
9952 */
9953 if (fStepping)
9954 {
9955 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9956 | CPUMCTX_EXTRN_CS);
9957 AssertRC(rc);
9958 if ( pCtx->rip != DbgState.uRipStart
9959 || pCtx->cs.Sel != DbgState.uCsStart)
9960 {
9961 rcStrict = VINF_EM_DBG_STEPPED;
9962 break;
9963 }
9964 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9965 }
9966
9967 /*
9968 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9969 */
9970 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9971 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9972 }
9973
9974 /*
9975 * Clear the X86_EFL_TF if necessary.
9976 */
9977 if (pVCpu->hm.s.fClearTrapFlag)
9978 {
9979 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9980 AssertRC(rc);
9981 pVCpu->hm.s.fClearTrapFlag = false;
9982 pCtx->eflags.Bits.u1TF = 0;
9983 }
9984 /** @todo there seems to be issues with the resume flag when the monitor trap
9985 * flag is pending without being used. Seen early in bios init when
9986 * accessing APIC page in protected mode. */
9987
9988 /*
9989 * Restore VM-exit control settings as we may not reenter this function the
9990 * next time around.
9991 */
9992 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9993
9994 /* Restore HMCPU indicators. */
9995 pVCpu->hm.s.fUsingDebugLoop = false;
9996 pVCpu->hm.s.fDebugWantRdTscExit = false;
9997 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9998
9999 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
10000 return rcStrict;
10001}
10002
10003
10004/** @} */
10005
10006
10007/**
10008 * Checks if any expensive dtrace probes are enabled and we should go to the
10009 * debug loop.
10010 *
10011 * @returns true if we should use debug loop, false if not.
10012 */
10013static bool hmR0VmxAnyExpensiveProbesEnabled(void)
10014{
10015 /* It's probably faster to OR the raw 32-bit counter variables together.
10016 Since the variables are in an array and the probes are next to one
10017 another (more or less), we have good locality. So, better read
10018 eight-nine cache lines ever time and only have one conditional, than
10019 128+ conditionals, right? */
10020 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
10021 | VBOXVMM_XCPT_DE_ENABLED_RAW()
10022 | VBOXVMM_XCPT_DB_ENABLED_RAW()
10023 | VBOXVMM_XCPT_BP_ENABLED_RAW()
10024 | VBOXVMM_XCPT_OF_ENABLED_RAW()
10025 | VBOXVMM_XCPT_BR_ENABLED_RAW()
10026 | VBOXVMM_XCPT_UD_ENABLED_RAW()
10027 | VBOXVMM_XCPT_NM_ENABLED_RAW()
10028 | VBOXVMM_XCPT_DF_ENABLED_RAW()
10029 | VBOXVMM_XCPT_TS_ENABLED_RAW()
10030 | VBOXVMM_XCPT_NP_ENABLED_RAW()
10031 | VBOXVMM_XCPT_SS_ENABLED_RAW()
10032 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10033 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10034 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10035 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10036 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10037 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10038 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10039 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10040 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10041 ) != 0
10042 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10043 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10044 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10045 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10046 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10047 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10048 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10049 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10050 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10051 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10052 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10053 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10054 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10055 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10056 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10057 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10058 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10059 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10060 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10061 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10062 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10063 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10064 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10065 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10066 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10067 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10068 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10069 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10070 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10071 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10072 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10073 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10074 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10075 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10076 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10077 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10078 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10079 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10080 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10081 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10082 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10083 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10084 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10085 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10086 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10087 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10088 ) != 0
10089 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10090 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10091 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10092 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10093 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10094 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10095 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10096 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10097 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10098 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10099 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10100 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10101 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10102 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10103 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10104 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10105 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10106 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10107 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10108 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10109 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10110 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10111 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10112 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10113 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10114 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10115 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10116 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10117 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10118 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10119 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10120 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10121 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10122 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10123 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10124 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10125 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10126 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10127 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10128 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10129 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10130 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10131 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10132 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10133 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10134 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10135 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10136 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10137 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10138 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10139 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10140 ) != 0;
10141}
10142
10143
10144/**
10145 * Runs the guest code using VT-x.
10146 *
10147 * @returns Strict VBox status code (i.e. informational status codes too).
10148 * @param pVCpu The cross context virtual CPU structure.
10149 * @param pCtx Pointer to the guest-CPU context.
10150 */
10151VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10152{
10153 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10154 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10155 HMVMX_ASSERT_PREEMPT_SAFE();
10156
10157 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10158
10159 VBOXSTRICTRC rcStrict;
10160 if ( !pVCpu->hm.s.fUseDebugLoop
10161 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10162 && !DBGFIsStepping(pVCpu)
10163 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10164 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10165 else
10166 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10167
10168 if (rcStrict == VERR_EM_INTERPRETER)
10169 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10170 else if (rcStrict == VINF_EM_RESET)
10171 rcStrict = VINF_EM_TRIPLE_FAULT;
10172
10173 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10174 if (RT_FAILURE(rc2))
10175 {
10176 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10177 rcStrict = rc2;
10178 }
10179 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10180 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10181 return rcStrict;
10182}
10183
10184
10185#ifndef HMVMX_USE_FUNCTION_TABLE
10186DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10187{
10188#ifdef DEBUG_ramshankar
10189#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10190 do { \
10191 if (a_fSave != 0) \
10192 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10193 VBOXSTRICTRC rcStrict = a_CallExpr; \
10194 if (a_fSave != 0) \
10195 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10196 return rcStrict; \
10197 } while (0)
10198#else
10199# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10200#endif
10201 switch (rcReason)
10202 {
10203 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10204 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10205 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10206 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10207 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10208 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10209 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10210 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10211 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10212 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10213 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10214 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10215 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10216 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10217 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10218 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10219 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10220 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10221 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10222 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10223 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10224 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10225 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10226 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10227 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10228 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10229 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10230 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10231 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10232 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10233 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10234 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10235 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10236 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10237
10238 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10239 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10240 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10241 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10242 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10243 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10244 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10245 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10246 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10247
10248 case VMX_EXIT_VMCLEAR:
10249 case VMX_EXIT_VMLAUNCH:
10250 case VMX_EXIT_VMPTRLD:
10251 case VMX_EXIT_VMPTRST:
10252 case VMX_EXIT_VMREAD:
10253 case VMX_EXIT_VMRESUME:
10254 case VMX_EXIT_VMWRITE:
10255 case VMX_EXIT_VMXOFF:
10256 case VMX_EXIT_VMXON:
10257 case VMX_EXIT_INVEPT:
10258 case VMX_EXIT_INVVPID:
10259 case VMX_EXIT_VMFUNC:
10260 case VMX_EXIT_XSAVES:
10261 case VMX_EXIT_XRSTORS:
10262 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10263
10264 case VMX_EXIT_ENCLS:
10265 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10266 case VMX_EXIT_PML_FULL:
10267 default:
10268 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10269 }
10270#undef VMEXIT_CALL_RET
10271}
10272#endif /* !HMVMX_USE_FUNCTION_TABLE */
10273
10274
10275#ifdef VBOX_STRICT
10276/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10277# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10278 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10279
10280# define HMVMX_ASSERT_PREEMPT_CPUID() \
10281 do { \
10282 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10283 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10284 } while (0)
10285
10286# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10287 do { \
10288 AssertPtr(pVCpu); \
10289 AssertPtr(pMixedCtx); \
10290 AssertPtr(pVmxTransient); \
10291 Assert(pVmxTransient->fVMEntryFailed == false); \
10292 Assert(ASMIntAreEnabled()); \
10293 HMVMX_ASSERT_PREEMPT_SAFE(); \
10294 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10295 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10296 HMVMX_ASSERT_PREEMPT_SAFE(); \
10297 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10298 HMVMX_ASSERT_PREEMPT_CPUID(); \
10299 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10300 } while (0)
10301
10302# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10303 do { \
10304 Log4Func(("\n")); \
10305 } while (0)
10306#else /* nonstrict builds: */
10307# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10308 do { \
10309 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10310 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10311 } while (0)
10312# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10313#endif
10314
10315
10316/**
10317 * Advances the guest RIP by the specified number of bytes.
10318 *
10319 * @param pVCpu The cross context virtual CPU structure.
10320 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10321 * out-of-sync. Make sure to update the required fields
10322 * before using them.
10323 * @param cbInstr Number of bytes to advance the RIP by.
10324 *
10325 * @remarks No-long-jump zone!!!
10326 */
10327DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10328{
10329 /* Advance the RIP. */
10330 pMixedCtx->rip += cbInstr;
10331 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10332
10333 /* Update interrupt inhibition. */
10334 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10335 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10336 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10337}
10338
10339
10340/**
10341 * Advances the guest RIP after reading it from the VMCS.
10342 *
10343 * @returns VBox status code, no informational status codes.
10344 * @param pVCpu The cross context virtual CPU structure.
10345 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10346 * out-of-sync. Make sure to update the required fields
10347 * before using them.
10348 * @param pVmxTransient Pointer to the VMX transient structure.
10349 *
10350 * @remarks No-long-jump zone!!!
10351 */
10352static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10353{
10354 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10355 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10356 | CPUMCTX_EXTRN_RFLAGS);
10357 AssertRCReturn(rc, rc);
10358
10359 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10360
10361 /*
10362 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10363 * pending debug exception field as it takes care of priority of events.
10364 *
10365 * See Intel spec. 32.2.1 "Debug Exceptions".
10366 */
10367 if ( !pVCpu->hm.s.fSingleInstruction
10368 && pMixedCtx->eflags.Bits.u1TF)
10369 {
10370 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10371 AssertRCReturn(rc, rc);
10372 }
10373
10374 return VINF_SUCCESS;
10375}
10376
10377
10378/**
10379 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10380 * and update error record fields accordingly.
10381 *
10382 * @return VMX_IGS_* return codes.
10383 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10384 * wrong with the guest state.
10385 *
10386 * @param pVCpu The cross context virtual CPU structure.
10387 * @param pCtx Pointer to the guest-CPU state.
10388 *
10389 * @remarks This function assumes our cache of the VMCS controls
10390 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10391 */
10392static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10393{
10394#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10395#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10396 uError = (err); \
10397 break; \
10398 } else do { } while (0)
10399
10400 int rc;
10401 PVM pVM = pVCpu->CTX_SUFF(pVM);
10402 uint32_t uError = VMX_IGS_ERROR;
10403 uint32_t u32Val;
10404 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10405
10406 do
10407 {
10408 /*
10409 * CR0.
10410 */
10411 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10412 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10413 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10414 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10415 if (fUnrestrictedGuest)
10416 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10417
10418 uint32_t u32GuestCr0;
10419 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10420 AssertRCBreak(rc);
10421 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10422 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10423 if ( !fUnrestrictedGuest
10424 && (u32GuestCr0 & X86_CR0_PG)
10425 && !(u32GuestCr0 & X86_CR0_PE))
10426 {
10427 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10428 }
10429
10430 /*
10431 * CR4.
10432 */
10433 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10434 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10435
10436 uint32_t u32GuestCr4;
10437 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10438 AssertRCBreak(rc);
10439 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10440 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10441
10442 /*
10443 * IA32_DEBUGCTL MSR.
10444 */
10445 uint64_t u64Val;
10446 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10447 AssertRCBreak(rc);
10448 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10449 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10450 {
10451 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10452 }
10453 uint64_t u64DebugCtlMsr = u64Val;
10454
10455#ifdef VBOX_STRICT
10456 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10457 AssertRCBreak(rc);
10458 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10459#endif
10460 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10461
10462 /*
10463 * RIP and RFLAGS.
10464 */
10465 uint32_t u32Eflags;
10466#if HC_ARCH_BITS == 64
10467 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10468 AssertRCBreak(rc);
10469 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10470 if ( !fLongModeGuest
10471 || !pCtx->cs.Attr.n.u1Long)
10472 {
10473 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10474 }
10475 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10476 * must be identical if the "IA-32e mode guest" VM-entry
10477 * control is 1 and CS.L is 1. No check applies if the
10478 * CPU supports 64 linear-address bits. */
10479
10480 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10481 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10482 AssertRCBreak(rc);
10483 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10484 VMX_IGS_RFLAGS_RESERVED);
10485 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10486 u32Eflags = u64Val;
10487#else
10488 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10489 AssertRCBreak(rc);
10490 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10491 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10492#endif
10493
10494 if ( fLongModeGuest
10495 || ( fUnrestrictedGuest
10496 && !(u32GuestCr0 & X86_CR0_PE)))
10497 {
10498 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10499 }
10500
10501 uint32_t u32EntryInfo;
10502 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10503 AssertRCBreak(rc);
10504 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10505 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10506 {
10507 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10508 }
10509
10510 /*
10511 * 64-bit checks.
10512 */
10513#if HC_ARCH_BITS == 64
10514 if (fLongModeGuest)
10515 {
10516 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10517 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10518 }
10519
10520 if ( !fLongModeGuest
10521 && (u32GuestCr4 & X86_CR4_PCIDE))
10522 {
10523 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10524 }
10525
10526 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10527 * 51:32 beyond the processor's physical-address width are 0. */
10528
10529 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10530 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10531 {
10532 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10533 }
10534
10535 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10536 AssertRCBreak(rc);
10537 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10538
10539 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10540 AssertRCBreak(rc);
10541 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10542#endif
10543
10544 /*
10545 * PERF_GLOBAL MSR.
10546 */
10547 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10548 {
10549 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10550 AssertRCBreak(rc);
10551 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10552 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10553 }
10554
10555 /*
10556 * PAT MSR.
10557 */
10558 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10559 {
10560 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10561 AssertRCBreak(rc);
10562 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10563 for (unsigned i = 0; i < 8; i++)
10564 {
10565 uint8_t u8Val = (u64Val & 0xff);
10566 if ( u8Val != 0 /* UC */
10567 && u8Val != 1 /* WC */
10568 && u8Val != 4 /* WT */
10569 && u8Val != 5 /* WP */
10570 && u8Val != 6 /* WB */
10571 && u8Val != 7 /* UC- */)
10572 {
10573 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10574 }
10575 u64Val >>= 8;
10576 }
10577 }
10578
10579 /*
10580 * EFER MSR.
10581 */
10582 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10583 {
10584 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10585 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10586 AssertRCBreak(rc);
10587 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10588 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10589 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10590 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10591 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10592 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10593 || !(u32GuestCr0 & X86_CR0_PG)
10594 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10595 VMX_IGS_EFER_LMA_LME_MISMATCH);
10596 }
10597
10598 /*
10599 * Segment registers.
10600 */
10601 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10602 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10603 if (!(u32Eflags & X86_EFL_VM))
10604 {
10605 /* CS */
10606 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10607 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10608 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10609 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10610 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10611 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10612 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10613 /* CS cannot be loaded with NULL in protected mode. */
10614 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10615 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10616 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10617 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10618 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10619 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10620 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10621 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10622 else
10623 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10624
10625 /* SS */
10626 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10627 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10628 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10629 if ( !(pCtx->cr0 & X86_CR0_PE)
10630 || pCtx->cs.Attr.n.u4Type == 3)
10631 {
10632 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10633 }
10634 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10635 {
10636 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10637 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10638 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10639 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10640 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10641 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10642 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10643 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10644 }
10645
10646 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10647 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10648 {
10649 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10650 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10651 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10652 || pCtx->ds.Attr.n.u4Type > 11
10653 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10654 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10655 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10656 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10657 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10658 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10659 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10660 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10661 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10662 }
10663 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10664 {
10665 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10666 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10667 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10668 || pCtx->es.Attr.n.u4Type > 11
10669 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10670 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10671 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10672 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10673 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10674 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10675 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10676 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10677 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10678 }
10679 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10680 {
10681 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10682 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10683 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10684 || pCtx->fs.Attr.n.u4Type > 11
10685 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10686 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10687 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10688 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10689 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10690 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10691 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10692 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10693 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10694 }
10695 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10696 {
10697 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10698 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10699 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10700 || pCtx->gs.Attr.n.u4Type > 11
10701 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10702 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10703 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10704 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10705 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10706 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10707 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10708 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10709 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10710 }
10711 /* 64-bit capable CPUs. */
10712#if HC_ARCH_BITS == 64
10713 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10714 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10715 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10716 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10717 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10718 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10719 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10720 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10721 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10722 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10723 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10724#endif
10725 }
10726 else
10727 {
10728 /* V86 mode checks. */
10729 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10730 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10731 {
10732 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10733 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10734 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10735 }
10736 else
10737 {
10738 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10739 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10740 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10741 }
10742
10743 /* CS */
10744 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10745 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10746 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10747 /* SS */
10748 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10749 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10750 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10751 /* DS */
10752 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10753 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10754 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10755 /* ES */
10756 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10757 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10758 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10759 /* FS */
10760 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10761 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10762 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10763 /* GS */
10764 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10765 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10766 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10767 /* 64-bit capable CPUs. */
10768#if HC_ARCH_BITS == 64
10769 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10770 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10771 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10772 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10773 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10774 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10775 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10776 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10777 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10778 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10779 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10780#endif
10781 }
10782
10783 /*
10784 * TR.
10785 */
10786 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10787 /* 64-bit capable CPUs. */
10788#if HC_ARCH_BITS == 64
10789 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10790#endif
10791 if (fLongModeGuest)
10792 {
10793 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10794 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10795 }
10796 else
10797 {
10798 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10799 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10800 VMX_IGS_TR_ATTR_TYPE_INVALID);
10801 }
10802 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10803 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10804 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10805 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10806 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10807 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10808 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10809 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10810
10811 /*
10812 * GDTR and IDTR.
10813 */
10814#if HC_ARCH_BITS == 64
10815 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10816 AssertRCBreak(rc);
10817 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10818
10819 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10820 AssertRCBreak(rc);
10821 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10822#endif
10823
10824 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10825 AssertRCBreak(rc);
10826 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10827
10828 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10829 AssertRCBreak(rc);
10830 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10831
10832 /*
10833 * Guest Non-Register State.
10834 */
10835 /* Activity State. */
10836 uint32_t u32ActivityState;
10837 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10838 AssertRCBreak(rc);
10839 HMVMX_CHECK_BREAK( !u32ActivityState
10840 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10841 VMX_IGS_ACTIVITY_STATE_INVALID);
10842 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10843 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10844 uint32_t u32IntrState;
10845 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10846 AssertRCBreak(rc);
10847 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10848 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10849 {
10850 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10851 }
10852
10853 /** @todo Activity state and injecting interrupts. Left as a todo since we
10854 * currently don't use activity states but ACTIVE. */
10855
10856 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10857 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10858
10859 /* Guest interruptibility-state. */
10860 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10861 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10862 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10863 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10864 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10865 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10866 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10867 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10868 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10869 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10870 {
10871 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10872 {
10873 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10874 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10875 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10876 }
10877 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10878 {
10879 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10880 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10881 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10882 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10883 }
10884 }
10885 /** @todo Assumes the processor is not in SMM. */
10886 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10887 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10888 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10889 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10890 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10891 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10892 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10893 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10894 {
10895 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10896 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10897 }
10898
10899 /* Pending debug exceptions. */
10900#if HC_ARCH_BITS == 64
10901 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10902 AssertRCBreak(rc);
10903 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10904 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10905 u32Val = u64Val; /* For pending debug exceptions checks below. */
10906#else
10907 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10908 AssertRCBreak(rc);
10909 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10910 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10911#endif
10912
10913 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10914 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10915 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10916 {
10917 if ( (u32Eflags & X86_EFL_TF)
10918 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10919 {
10920 /* Bit 14 is PendingDebug.BS. */
10921 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10922 }
10923 if ( !(u32Eflags & X86_EFL_TF)
10924 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10925 {
10926 /* Bit 14 is PendingDebug.BS. */
10927 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10928 }
10929 }
10930
10931 /* VMCS link pointer. */
10932 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10933 AssertRCBreak(rc);
10934 if (u64Val != UINT64_C(0xffffffffffffffff))
10935 {
10936 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10937 /** @todo Bits beyond the processor's physical-address width MBZ. */
10938 /** @todo 32-bit located in memory referenced by value of this field (as a
10939 * physical address) must contain the processor's VMCS revision ID. */
10940 /** @todo SMM checks. */
10941 }
10942
10943 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10944 * not using Nested Paging? */
10945 if ( pVM->hm.s.fNestedPaging
10946 && !fLongModeGuest
10947 && CPUMIsGuestInPAEModeEx(pCtx))
10948 {
10949 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10950 AssertRCBreak(rc);
10951 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10952
10953 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10954 AssertRCBreak(rc);
10955 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10956
10957 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10958 AssertRCBreak(rc);
10959 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10960
10961 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10962 AssertRCBreak(rc);
10963 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10964 }
10965
10966 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10967 if (uError == VMX_IGS_ERROR)
10968 uError = VMX_IGS_REASON_NOT_FOUND;
10969 } while (0);
10970
10971 pVCpu->hm.s.u32HMError = uError;
10972 return uError;
10973
10974#undef HMVMX_ERROR_BREAK
10975#undef HMVMX_CHECK_BREAK
10976}
10977
10978/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10979/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10980/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10981
10982/** @name VM-exit handlers.
10983 * @{
10984 */
10985
10986/**
10987 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10988 */
10989HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10990{
10991 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10992 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10993 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10994 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10995 return VINF_SUCCESS;
10996 return VINF_EM_RAW_INTERRUPT;
10997}
10998
10999
11000/**
11001 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
11002 */
11003HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11004{
11005 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11006 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
11007
11008 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
11009 AssertRCReturn(rc, rc);
11010
11011 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
11012 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
11013 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
11014 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
11015
11016 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
11017 {
11018 /*
11019 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
11020 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
11021 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
11022 *
11023 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
11024 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
11025 */
11026 VMXDispatchHostNmi();
11027 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
11028 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11029 return VINF_SUCCESS;
11030 }
11031
11032 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11033 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11034 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11035 { /* likely */ }
11036 else
11037 {
11038 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11039 rcStrictRc1 = VINF_SUCCESS;
11040 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11041 return rcStrictRc1;
11042 }
11043
11044 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11045 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11046 switch (uIntType)
11047 {
11048 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11049 Assert(uVector == X86_XCPT_DB);
11050 RT_FALL_THRU();
11051 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11052 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11053 RT_FALL_THRU();
11054 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11055 {
11056 /*
11057 * If there's any exception caused as a result of event injection, the resulting
11058 * secondary/final execption will be pending, we shall continue guest execution
11059 * after injecting the event. The page-fault case is complicated and we manually
11060 * handle any currently pending event in hmR0VmxExitXcptPF.
11061 */
11062 if (!pVCpu->hm.s.Event.fPending)
11063 { /* likely */ }
11064 else if (uVector != X86_XCPT_PF)
11065 {
11066 rc = VINF_SUCCESS;
11067 break;
11068 }
11069
11070 switch (uVector)
11071 {
11072 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11073 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11074 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11075 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11076 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11077 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11078
11079 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11080 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11081 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11082 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11083 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11084 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11085 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11086 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11087 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11088 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11089 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11090 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11091 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11092 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11093 default:
11094 {
11095 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11096 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11097 {
11098 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11099 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11100 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11101
11102 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11103 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11104 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11105 AssertRCReturn(rc, rc);
11106 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11107 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11108 0 /* GCPtrFaultAddress */);
11109 }
11110 else
11111 {
11112 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11113 pVCpu->hm.s.u32HMError = uVector;
11114 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11115 }
11116 break;
11117 }
11118 }
11119 break;
11120 }
11121
11122 default:
11123 {
11124 pVCpu->hm.s.u32HMError = uExitIntInfo;
11125 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11126 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11127 break;
11128 }
11129 }
11130 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11131 return rc;
11132}
11133
11134
11135/**
11136 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11137 */
11138HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11139{
11140 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11141
11142 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11143 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11144
11145 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11147 return VINF_SUCCESS;
11148}
11149
11150
11151/**
11152 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11153 */
11154HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11155{
11156 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11157 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11158 {
11159 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11160 HMVMX_RETURN_UNEXPECTED_EXIT();
11161 }
11162
11163 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11164
11165 /*
11166 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11167 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11168 */
11169 uint32_t fIntrState = 0;
11170 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11171 AssertRCReturn(rc, rc);
11172
11173 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11174 if ( fBlockSti
11175 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11176 {
11177 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11178 }
11179
11180 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11181 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11182
11183 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11184 return VINF_SUCCESS;
11185}
11186
11187
11188/**
11189 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11190 */
11191HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11192{
11193 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11194 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11195}
11196
11197
11198/**
11199 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11200 */
11201HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11202{
11203 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11204 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11205}
11206
11207
11208/**
11209 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11210 */
11211HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11212{
11213 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11214 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11215
11216 /*
11217 * Get the state we need and update the exit history entry.
11218 */
11219 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11220 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11221 | CPUMCTX_EXTRN_CS);
11222 AssertRCReturn(rc, rc);
11223
11224 VBOXSTRICTRC rcStrict;
11225 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11226 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11227 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11228 if (!pExitRec)
11229 {
11230 /*
11231 * Regular CPUID instruction execution.
11232 */
11233 PVM pVM = pVCpu->CTX_SUFF(pVM);
11234 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11235 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11236 {
11237 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11238 Assert(pVmxTransient->cbInstr == 2);
11239 }
11240 else
11241 {
11242 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11243 rcStrict = VERR_EM_INTERPRETER;
11244 }
11245 }
11246 else
11247 {
11248 /*
11249 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11250 */
11251 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11252 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11253 AssertRCReturn(rc2, rc2);
11254
11255 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11256 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11257
11258 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11259 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11260
11261 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11262 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11263 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11264 }
11265 return VBOXSTRICTRC_TODO(rcStrict);
11266}
11267
11268
11269/**
11270 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11271 */
11272HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11273{
11274 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11275 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11276 AssertRCReturn(rc, rc);
11277
11278 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11279 return VINF_EM_RAW_EMULATE_INSTR;
11280
11281 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11282 HMVMX_RETURN_UNEXPECTED_EXIT();
11283}
11284
11285
11286/**
11287 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11288 */
11289HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11290{
11291 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11292 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11293 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11294 AssertRCReturn(rc, rc);
11295
11296 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11297 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11298 {
11299 /* If we get a spurious VM-exit when offsetting is enabled,
11300 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11301 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11302 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11303 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11304 | HM_CHANGED_GUEST_RFLAGS);
11305 }
11306 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11307 {
11308 rcStrict = VINF_SUCCESS;
11309 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11310 }
11311 return rcStrict;
11312}
11313
11314
11315/**
11316 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11317 */
11318HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11319{
11320 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11321 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11322 | CPUMCTX_EXTRN_TSC_AUX);
11323 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11324 AssertRCReturn(rc, rc);
11325
11326 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11327 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11328 {
11329 /* If we get a spurious VM-exit when offsetting is enabled,
11330 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11331 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11332 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11333 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11334 | HM_CHANGED_GUEST_RFLAGS);
11335 }
11336 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11337 {
11338 rcStrict = VINF_SUCCESS;
11339 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11340 }
11341 return rcStrict;
11342}
11343
11344
11345/**
11346 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11347 */
11348HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11349{
11350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11351 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11352 | CPUMCTX_EXTRN_CR0
11353 | CPUMCTX_EXTRN_RFLAGS
11354 | CPUMCTX_EXTRN_SS);
11355 AssertRCReturn(rc, rc);
11356
11357 PVM pVM = pVCpu->CTX_SUFF(pVM);
11358 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11359 if (RT_LIKELY(rc == VINF_SUCCESS))
11360 {
11361 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11362 Assert(pVmxTransient->cbInstr == 2);
11363 }
11364 else
11365 {
11366 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11367 rc = VERR_EM_INTERPRETER;
11368 }
11369 return rc;
11370}
11371
11372
11373/**
11374 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11375 */
11376HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11377{
11378 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11379
11380 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11381 if (EMAreHypercallInstructionsEnabled(pVCpu))
11382 {
11383 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11384 | CPUMCTX_EXTRN_RFLAGS
11385 | CPUMCTX_EXTRN_CR0
11386 | CPUMCTX_EXTRN_SS
11387 | CPUMCTX_EXTRN_CS
11388 | CPUMCTX_EXTRN_EFER);
11389 AssertRCReturn(rc, rc);
11390
11391 /* Perform the hypercall. */
11392 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11393 if (rcStrict == VINF_SUCCESS)
11394 {
11395 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11396 AssertRCReturn(rc, rc);
11397 }
11398 else
11399 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11400 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11401 || RT_FAILURE(rcStrict));
11402
11403 /* If the hypercall changes anything other than guest's general-purpose registers,
11404 we would need to reload the guest changed bits here before VM-entry. */
11405 }
11406 else
11407 Log4Func(("Hypercalls not enabled\n"));
11408
11409 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11410 if (RT_FAILURE(rcStrict))
11411 {
11412 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11413 rcStrict = VINF_SUCCESS;
11414 }
11415
11416 return rcStrict;
11417}
11418
11419
11420/**
11421 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11422 */
11423HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11424{
11425 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11426 PVM pVM = pVCpu->CTX_SUFF(pVM);
11427 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11428
11429 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11430 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11431 AssertRCReturn(rc, rc);
11432
11433 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11434 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11435 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11436 else
11437 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11438 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11439 return rcStrict;
11440}
11441
11442
11443/**
11444 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11445 */
11446HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11447{
11448 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11449 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11450 | CPUMCTX_EXTRN_RFLAGS
11451 | CPUMCTX_EXTRN_SS);
11452 AssertRCReturn(rc, rc);
11453
11454 PVM pVM = pVCpu->CTX_SUFF(pVM);
11455 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11456 if (RT_LIKELY(rc == VINF_SUCCESS))
11457 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11458 else
11459 {
11460 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11461 rc = VERR_EM_INTERPRETER;
11462 }
11463 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11464 return rc;
11465}
11466
11467
11468/**
11469 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11470 */
11471HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11472{
11473 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11474 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11475 | CPUMCTX_EXTRN_RFLAGS
11476 | CPUMCTX_EXTRN_SS);
11477 AssertRCReturn(rc, rc);
11478
11479 PVM pVM = pVCpu->CTX_SUFF(pVM);
11480 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11481 rc = VBOXSTRICTRC_VAL(rc2);
11482 if (RT_LIKELY( rc == VINF_SUCCESS
11483 || rc == VINF_EM_HALT))
11484 {
11485 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11486 AssertRCReturn(rc3, rc3);
11487
11488 if ( rc == VINF_EM_HALT
11489 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11490 rc = VINF_SUCCESS;
11491 }
11492 else
11493 {
11494 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11495 rc = VERR_EM_INTERPRETER;
11496 }
11497 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11498 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11499 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11500 return rc;
11501}
11502
11503
11504/**
11505 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11506 */
11507HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11508{
11509 /*
11510 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11511 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11512 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11513 * VMX root operation. If we get here, something funny is going on.
11514 *
11515 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11516 */
11517 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11518 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11519 HMVMX_RETURN_UNEXPECTED_EXIT();
11520}
11521
11522
11523/**
11524 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11525 */
11526HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11527{
11528 /*
11529 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11530 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11531 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11532 * an SMI. If we get here, something funny is going on.
11533 *
11534 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11535 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11536 */
11537 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11538 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11539 HMVMX_RETURN_UNEXPECTED_EXIT();
11540}
11541
11542
11543/**
11544 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11545 */
11546HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11547{
11548 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11549 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11550 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11551 HMVMX_RETURN_UNEXPECTED_EXIT();
11552}
11553
11554
11555/**
11556 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11557 */
11558HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11559{
11560 /*
11561 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11562 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11563 * See Intel spec. 25.3 "Other Causes of VM-exits".
11564 */
11565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11566 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11567 HMVMX_RETURN_UNEXPECTED_EXIT();
11568}
11569
11570
11571/**
11572 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11573 * VM-exit.
11574 */
11575HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11576{
11577 /*
11578 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11579 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11580 *
11581 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11582 * See Intel spec. "23.8 Restrictions on VMX operation".
11583 */
11584 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11585 return VINF_SUCCESS;
11586}
11587
11588
11589/**
11590 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11591 * VM-exit.
11592 */
11593HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11594{
11595 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11596 return VINF_EM_RESET;
11597}
11598
11599
11600/**
11601 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11602 */
11603HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11604{
11605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11606 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11607
11608 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11609 AssertRCReturn(rc, rc);
11610
11611 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11612 rc = VINF_SUCCESS;
11613 else
11614 rc = VINF_EM_HALT;
11615
11616 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11617 if (rc != VINF_SUCCESS)
11618 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11619 return rc;
11620}
11621
11622
11623/**
11624 * VM-exit handler for instructions that result in a \#UD exception delivered to
11625 * the guest.
11626 */
11627HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11628{
11629 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11630 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11631 return VINF_SUCCESS;
11632}
11633
11634
11635/**
11636 * VM-exit handler for expiry of the VMX preemption timer.
11637 */
11638HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11639{
11640 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11641
11642 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11643 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11644
11645 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11646 PVM pVM = pVCpu->CTX_SUFF(pVM);
11647 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11648 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11649 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11650}
11651
11652
11653/**
11654 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11655 */
11656HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11657{
11658 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11659
11660 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11661 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11662 | CPUMCTX_EXTRN_CR4);
11663 AssertRCReturn(rc, rc);
11664
11665 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11666 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11667 : HM_CHANGED_XCPT_RAISED_MASK);
11668
11669 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11670
11671 return rcStrict;
11672}
11673
11674
11675/**
11676 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11677 */
11678HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11679{
11680 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11681 /** @todo Use VM-exit instruction information. */
11682 return VERR_EM_INTERPRETER;
11683}
11684
11685
11686/**
11687 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11688 * Error VM-exit.
11689 */
11690HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11691{
11692 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11693 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11694 AssertRCReturn(rc, rc);
11695
11696 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11697 NOREF(uInvalidReason);
11698
11699#ifdef VBOX_STRICT
11700 uint32_t fIntrState;
11701 RTHCUINTREG uHCReg;
11702 uint64_t u64Val;
11703 uint32_t u32Val;
11704
11705 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11706 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11707 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11708 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11709 AssertRCReturn(rc, rc);
11710
11711 Log4(("uInvalidReason %u\n", uInvalidReason));
11712 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11713 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11714 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11715 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11716
11717 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11718 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11719 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11720 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11721 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11722 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11723 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11724 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11725 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11726 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11727 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11728 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11729
11730 hmR0DumpRegs(pVCpu, pMixedCtx);
11731#else
11732 NOREF(pVmxTransient);
11733#endif
11734
11735 return VERR_VMX_INVALID_GUEST_STATE;
11736}
11737
11738
11739/**
11740 * VM-exit handler for VM-entry failure due to an MSR-load
11741 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11742 */
11743HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11744{
11745 NOREF(pVmxTransient);
11746 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11747 HMVMX_RETURN_UNEXPECTED_EXIT();
11748}
11749
11750
11751/**
11752 * VM-exit handler for VM-entry failure due to a machine-check event
11753 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11754 */
11755HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11756{
11757 NOREF(pVmxTransient);
11758 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11759 HMVMX_RETURN_UNEXPECTED_EXIT();
11760}
11761
11762
11763/**
11764 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11765 * theory.
11766 */
11767HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11768{
11769 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11770 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11771 return VERR_VMX_UNDEFINED_EXIT_CODE;
11772}
11773
11774
11775/**
11776 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11777 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11778 * Conditional VM-exit.
11779 */
11780HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11781{
11782 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11783
11784 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11785 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11786 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11787 return VERR_EM_INTERPRETER;
11788 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11789 HMVMX_RETURN_UNEXPECTED_EXIT();
11790}
11791
11792
11793/**
11794 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11795 */
11796HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11797{
11798 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11799
11800 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11801 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11802 return VERR_EM_INTERPRETER;
11803 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11804 HMVMX_RETURN_UNEXPECTED_EXIT();
11805}
11806
11807
11808/**
11809 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11810 */
11811HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11812{
11813 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11814
11815 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11816 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11817 | CPUMCTX_EXTRN_RFLAGS
11818 | CPUMCTX_EXTRN_SS
11819 | CPUMCTX_EXTRN_FS
11820 | CPUMCTX_EXTRN_GS);
11821 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11822 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11823 AssertRCReturn(rc, rc);
11824 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11825
11826#ifdef VBOX_STRICT
11827 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11828 {
11829 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11830 && pMixedCtx->ecx != MSR_K6_EFER)
11831 {
11832 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11833 pMixedCtx->ecx));
11834 HMVMX_RETURN_UNEXPECTED_EXIT();
11835 }
11836 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11837 {
11838 VMXMSREXITREAD enmRead;
11839 VMXMSREXITWRITE enmWrite;
11840 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11841 AssertRCReturn(rc2, rc2);
11842 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11843 {
11844 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11845 HMVMX_RETURN_UNEXPECTED_EXIT();
11846 }
11847 }
11848 }
11849#endif
11850
11851 PVM pVM = pVCpu->CTX_SUFF(pVM);
11852 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11853 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11854 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11855 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11856 if (RT_SUCCESS(rc))
11857 {
11858 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11859 Assert(pVmxTransient->cbInstr == 2);
11860 }
11861 return rc;
11862}
11863
11864
11865/**
11866 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11867 */
11868HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11869{
11870 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11871 PVM pVM = pVCpu->CTX_SUFF(pVM);
11872 int rc = VINF_SUCCESS;
11873
11874 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. FS, GS (base) can be accessed by MSR writes. */
11875 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11876 | CPUMCTX_EXTRN_RFLAGS
11877 | CPUMCTX_EXTRN_SS
11878 | CPUMCTX_EXTRN_FS
11879 | CPUMCTX_EXTRN_GS);
11880 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11881 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11882 AssertRCReturn(rc, rc);
11883 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11884
11885 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11886 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11887 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11888
11889 if (RT_SUCCESS(rc))
11890 {
11891 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11892
11893 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11894 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11895 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11896 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11897 {
11898 /*
11899 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11900 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11901 * EMInterpretWrmsr() changes it.
11902 */
11903 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11904 }
11905 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11906 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11907 else if (pMixedCtx->ecx == MSR_K6_EFER)
11908 {
11909 /*
11910 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11911 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11912 * the other bits as well, SCE and NXE. See @bugref{7368}.
11913 */
11914 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11915 | HM_CHANGED_VMX_ENTRY_CTLS
11916 | HM_CHANGED_VMX_EXIT_CTLS);
11917 }
11918
11919 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11920 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11921 {
11922 switch (pMixedCtx->ecx)
11923 {
11924 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11925 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11926 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11927 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11928 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11929 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11930 default:
11931 {
11932 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11933 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11934 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11935 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11936 break;
11937 }
11938 }
11939 }
11940#ifdef VBOX_STRICT
11941 else
11942 {
11943 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11944 switch (pMixedCtx->ecx)
11945 {
11946 case MSR_IA32_SYSENTER_CS:
11947 case MSR_IA32_SYSENTER_EIP:
11948 case MSR_IA32_SYSENTER_ESP:
11949 case MSR_K8_FS_BASE:
11950 case MSR_K8_GS_BASE:
11951 {
11952 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11953 HMVMX_RETURN_UNEXPECTED_EXIT();
11954 }
11955
11956 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11957 default:
11958 {
11959 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11960 {
11961 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11962 if (pMixedCtx->ecx != MSR_K6_EFER)
11963 {
11964 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11965 pMixedCtx->ecx));
11966 HMVMX_RETURN_UNEXPECTED_EXIT();
11967 }
11968 }
11969
11970 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11971 {
11972 VMXMSREXITREAD enmRead;
11973 VMXMSREXITWRITE enmWrite;
11974 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11975 AssertRCReturn(rc2, rc2);
11976 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11977 {
11978 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11979 HMVMX_RETURN_UNEXPECTED_EXIT();
11980 }
11981 }
11982 break;
11983 }
11984 }
11985 }
11986#endif /* VBOX_STRICT */
11987 }
11988 return rc;
11989}
11990
11991
11992/**
11993 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11994 */
11995HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11996{
11997 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11998 /** @todo The guest has likely hit a contended spinlock. We might want to
11999 * poke a schedule different guest VCPU. */
12000 return VINF_EM_RAW_INTERRUPT;
12001}
12002
12003
12004/**
12005 * VM-exit handler for when the TPR value is lowered below the specified
12006 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
12007 */
12008HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12009{
12010 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12011 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
12012
12013 /*
12014 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
12015 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
12016 */
12017 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
12018 return VINF_SUCCESS;
12019}
12020
12021
12022/**
12023 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
12024 * VM-exit.
12025 *
12026 * @retval VINF_SUCCESS when guest execution can continue.
12027 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
12028 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
12029 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
12030 * interpreter.
12031 */
12032HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12033{
12034 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12035 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12036
12037 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12038 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12039 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12040 AssertRCReturn(rc, rc);
12041
12042 VBOXSTRICTRC rcStrict;
12043 PVM pVM = pVCpu->CTX_SUFF(pVM);
12044 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12045 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12046 switch (uAccessType)
12047 {
12048 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12049 {
12050 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12051 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12052 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12053 AssertMsg( rcStrict == VINF_SUCCESS
12054 || rcStrict == VINF_IEM_RAISED_XCPT
12055 || rcStrict == VINF_PGM_CHANGE_MODE
12056 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12057
12058 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12059 {
12060 case 0:
12061 {
12062 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12063 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12064 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12065 break;
12066 }
12067
12068 case 2:
12069 {
12070 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12071 /* Nothing to do here, CR2 it's not part of the VMCS. */
12072 break;
12073 }
12074
12075 case 3:
12076 {
12077 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12078 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12079 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12080 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12081 break;
12082 }
12083
12084 case 4:
12085 {
12086 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12087 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12088 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12089 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12090 break;
12091 }
12092
12093 case 8:
12094 {
12095 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12096 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12097 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12098 break;
12099 }
12100 default:
12101 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12102 break;
12103 }
12104 break;
12105 }
12106
12107 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12108 {
12109 Assert( !pVM->hm.s.fNestedPaging
12110 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12111 || pVCpu->hm.s.fUsingDebugLoop
12112 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12113 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12114 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12115 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12116
12117 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12118 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12119 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12120 AssertMsg( rcStrict == VINF_SUCCESS
12121 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12122#ifdef VBOX_WITH_STATISTICS
12123 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12124 {
12125 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12126 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12127 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12128 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12129 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12130 }
12131#endif
12132 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12133 VBOXSTRICTRC_VAL(rcStrict)));
12134 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12135 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12136 break;
12137 }
12138
12139 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12140 {
12141 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12142 AssertMsg( rcStrict == VINF_SUCCESS
12143 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12144
12145 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12146 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12147 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12148 break;
12149 }
12150
12151 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12152 {
12153 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12154 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12155 AssertMsg( rcStrict == VINF_SUCCESS
12156 || rcStrict == VINF_IEM_RAISED_XCPT
12157 || rcStrict == VINF_PGM_CHANGE_MODE,
12158 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12159
12160 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12161 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12162 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12163 break;
12164 }
12165
12166 default:
12167 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12168 VERR_VMX_UNEXPECTED_EXCEPTION);
12169 }
12170
12171 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12172 : HM_CHANGED_XCPT_RAISED_MASK);
12173 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12174 NOREF(pVM);
12175 return rcStrict;
12176}
12177
12178
12179/**
12180 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12181 * VM-exit.
12182 */
12183HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12184{
12185 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12186 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12187 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12188
12189 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12190 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12191 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12192 | CPUMCTX_EXTRN_SREG_MASK
12193 | CPUMCTX_EXTRN_EFER);
12194 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12195 AssertRCReturn(rc, rc);
12196
12197 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12198 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12199 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12200 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12201 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12202 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12203 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12204 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12205 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12206
12207 /*
12208 * Update exit history to see if this exit can be optimized.
12209 */
12210 VBOXSTRICTRC rcStrict;
12211 PCEMEXITREC pExitRec = NULL;
12212 if ( !fGstStepping
12213 && !fDbgStepping)
12214 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12215 !fIOString
12216 ? !fIOWrite
12217 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12218 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12219 : !fIOWrite
12220 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12221 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12222 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12223 if (!pExitRec)
12224 {
12225 /* I/O operation lookup arrays. */
12226 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12227 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12228 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12229 uint32_t const cbInstr = pVmxTransient->cbInstr;
12230 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12231 PVM pVM = pVCpu->CTX_SUFF(pVM);
12232 if (fIOString)
12233 {
12234 /*
12235 * INS/OUTS - I/O String instruction.
12236 *
12237 * Use instruction-information if available, otherwise fall back on
12238 * interpreting the instruction.
12239 */
12240 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12241 fIOWrite ? 'w' : 'r'));
12242 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12243 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12244 {
12245 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12246 AssertRCReturn(rc2, rc2);
12247 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12248 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12249 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12250 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12251 if (fIOWrite)
12252 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12253 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12254 else
12255 {
12256 /*
12257 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12258 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12259 * See Intel Instruction spec. for "INS".
12260 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12261 */
12262 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12263 }
12264 }
12265 else
12266 rcStrict = IEMExecOne(pVCpu);
12267
12268 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12269 fUpdateRipAlready = true;
12270 }
12271 else
12272 {
12273 /*
12274 * IN/OUT - I/O instruction.
12275 */
12276 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12277 fIOWrite ? 'w' : 'r'));
12278 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12279 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12280 if (fIOWrite)
12281 {
12282 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12283 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12284 }
12285 else
12286 {
12287 uint32_t u32Result = 0;
12288 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12289 if (IOM_SUCCESS(rcStrict))
12290 {
12291 /* Save result of I/O IN instr. in AL/AX/EAX. */
12292 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12293 }
12294 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12295 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12296 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12297 }
12298 }
12299
12300 if (IOM_SUCCESS(rcStrict))
12301 {
12302 if (!fUpdateRipAlready)
12303 {
12304 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12305 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12306 }
12307
12308 /*
12309 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12310 * while booting Fedora 17 64-bit guest.
12311 *
12312 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12313 */
12314 if (fIOString)
12315 {
12316 /** @todo Single-step for INS/OUTS with REP prefix? */
12317 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12318 }
12319 else if ( !fDbgStepping
12320 && fGstStepping)
12321 {
12322 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12323 AssertRCReturn(rc, rc);
12324 }
12325
12326 /*
12327 * If any I/O breakpoints are armed, we need to check if one triggered
12328 * and take appropriate action.
12329 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12330 */
12331 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12332 AssertRCReturn(rc, rc);
12333
12334 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12335 * execution engines about whether hyper BPs and such are pending. */
12336 uint32_t const uDr7 = pMixedCtx->dr[7];
12337 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12338 && X86_DR7_ANY_RW_IO(uDr7)
12339 && (pMixedCtx->cr4 & X86_CR4_DE))
12340 || DBGFBpIsHwIoArmed(pVM)))
12341 {
12342 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12343
12344 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12345 VMMRZCallRing3Disable(pVCpu);
12346 HM_DISABLE_PREEMPT();
12347
12348 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12349
12350 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12351 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12352 {
12353 /* Raise #DB. */
12354 if (fIsGuestDbgActive)
12355 ASMSetDR6(pMixedCtx->dr[6]);
12356 if (pMixedCtx->dr[7] != uDr7)
12357 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12358
12359 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12360 }
12361 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12362 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12363 else if ( rcStrict2 != VINF_SUCCESS
12364 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12365 rcStrict = rcStrict2;
12366 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12367
12368 HM_RESTORE_PREEMPT();
12369 VMMRZCallRing3Enable(pVCpu);
12370 }
12371 }
12372
12373#ifdef VBOX_STRICT
12374 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12375 Assert(!fIOWrite);
12376 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12377 Assert(fIOWrite);
12378 else
12379 {
12380# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12381 * statuses, that the VMM device and some others may return. See
12382 * IOM_SUCCESS() for guidance. */
12383 AssertMsg( RT_FAILURE(rcStrict)
12384 || rcStrict == VINF_SUCCESS
12385 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12386 || rcStrict == VINF_EM_DBG_BREAKPOINT
12387 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12388 || rcStrict == VINF_EM_RAW_TO_R3
12389 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12390# endif
12391 }
12392#endif
12393 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12394 }
12395 else
12396 {
12397 /*
12398 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12399 */
12400 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12401 AssertRCReturn(rc2, rc2);
12402 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12403 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12404 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12405 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12406 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12407 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12408
12409 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12410 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12411
12412 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12413 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12414 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12415 }
12416 return rcStrict;
12417}
12418
12419
12420/**
12421 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12422 * VM-exit.
12423 */
12424HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12425{
12426 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12427
12428 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12429 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12430 AssertRCReturn(rc, rc);
12431 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12432 {
12433 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12434 AssertRCReturn(rc, rc);
12435 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12436 {
12437 uint32_t uErrCode;
12438 RTGCUINTPTR GCPtrFaultAddress;
12439 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12440 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12441 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12442 if (fErrorCodeValid)
12443 {
12444 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12445 AssertRCReturn(rc, rc);
12446 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12447 }
12448 else
12449 uErrCode = 0;
12450
12451 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12452 && uVector == X86_XCPT_PF)
12453 GCPtrFaultAddress = pMixedCtx->cr2;
12454 else
12455 GCPtrFaultAddress = 0;
12456
12457 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12458 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12459
12460 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12462 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12463 }
12464 }
12465
12466 /* Fall back to the interpreter to emulate the task-switch. */
12467 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12468 return VERR_EM_INTERPRETER;
12469}
12470
12471
12472/**
12473 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12474 */
12475HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12476{
12477 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12478 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12479 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12480 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12481 AssertRCReturn(rc, rc);
12482 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12483 return VINF_EM_DBG_STEPPED;
12484}
12485
12486
12487/**
12488 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12489 */
12490HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12491{
12492 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12493
12494 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12495
12496 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12497 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12498 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12499 {
12500 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12501 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12502 {
12503 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12504 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12505 }
12506 }
12507 else
12508 {
12509 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12510 rcStrict1 = VINF_SUCCESS;
12511 return rcStrict1;
12512 }
12513
12514 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12515 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12516 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12517 AssertRCReturn(rc, rc);
12518
12519 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12520 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12521 VBOXSTRICTRC rcStrict2;
12522 switch (uAccessType)
12523 {
12524 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12525 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12526 {
12527 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12528 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12529 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12530
12531 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12532 GCPhys &= PAGE_BASE_GC_MASK;
12533 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12534 PVM pVM = pVCpu->CTX_SUFF(pVM);
12535 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12536 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12537
12538 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12539 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12540 CPUMCTX2CORE(pMixedCtx), GCPhys);
12541 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12542 if ( rcStrict2 == VINF_SUCCESS
12543 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12544 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12545 {
12546 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12547 | HM_CHANGED_GUEST_RSP
12548 | HM_CHANGED_GUEST_RFLAGS
12549 | HM_CHANGED_GUEST_APIC_TPR);
12550 rcStrict2 = VINF_SUCCESS;
12551 }
12552 break;
12553 }
12554
12555 default:
12556 Log4Func(("uAccessType=%#x\n", uAccessType));
12557 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12558 break;
12559 }
12560
12561 if (rcStrict2 != VINF_SUCCESS)
12562 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12563 return rcStrict2;
12564}
12565
12566
12567/**
12568 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12569 * VM-exit.
12570 */
12571HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12572{
12573 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12574
12575 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12576 if (pVmxTransient->fWasGuestDebugStateActive)
12577 {
12578 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12579 HMVMX_RETURN_UNEXPECTED_EXIT();
12580 }
12581
12582 if ( !pVCpu->hm.s.fSingleInstruction
12583 && !pVmxTransient->fWasHyperDebugStateActive)
12584 {
12585 Assert(!DBGFIsStepping(pVCpu));
12586 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12587
12588 /* Don't intercept MOV DRx any more. */
12589 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12590 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12591 AssertRCReturn(rc, rc);
12592
12593 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12594 VMMRZCallRing3Disable(pVCpu);
12595 HM_DISABLE_PREEMPT();
12596
12597 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12598 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12599 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12600
12601 HM_RESTORE_PREEMPT();
12602 VMMRZCallRing3Enable(pVCpu);
12603
12604#ifdef VBOX_WITH_STATISTICS
12605 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12606 AssertRCReturn(rc, rc);
12607 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12609 else
12610 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12611#endif
12612 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12613 return VINF_SUCCESS;
12614 }
12615
12616 /*
12617 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12618 * Update the segment registers and DR7 from the CPU.
12619 */
12620 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12621 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12622 | CPUMCTX_EXTRN_DR7);
12623 AssertRCReturn(rc, rc);
12624 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12625
12626 PVM pVM = pVCpu->CTX_SUFF(pVM);
12627 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12628 {
12629 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12630 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12631 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12632 if (RT_SUCCESS(rc))
12633 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12634 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12635 }
12636 else
12637 {
12638 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12639 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12640 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12641 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12642 }
12643
12644 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12645 if (RT_SUCCESS(rc))
12646 {
12647 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12648 AssertRCReturn(rc2, rc2);
12649 return VINF_SUCCESS;
12650 }
12651 return rc;
12652}
12653
12654
12655/**
12656 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12657 * Conditional VM-exit.
12658 */
12659HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12660{
12661 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12662 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12663
12664 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12665 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12666 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12667 {
12668 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12669 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12670 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12671 {
12672 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12673 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12674 }
12675 }
12676 else
12677 {
12678 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12679 rcStrict1 = VINF_SUCCESS;
12680 return rcStrict1;
12681 }
12682
12683 /*
12684 * Get sufficent state and update the exit history entry.
12685 */
12686 RTGCPHYS GCPhys;
12687 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12688 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12689 AssertRCReturn(rc, rc);
12690
12691 VBOXSTRICTRC rcStrict;
12692 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12693 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12694 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12695 if (!pExitRec)
12696 {
12697 /*
12698 * If we succeed, resume guest execution.
12699 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12700 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12701 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12702 * weird case. See @bugref{6043}.
12703 */
12704 PVM pVM = pVCpu->CTX_SUFF(pVM);
12705 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12706 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12707 if ( rcStrict == VINF_SUCCESS
12708 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12709 || rcStrict == VERR_PAGE_NOT_PRESENT)
12710 {
12711 /* Successfully handled MMIO operation. */
12712 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12713 | HM_CHANGED_GUEST_RSP
12714 | HM_CHANGED_GUEST_RFLAGS
12715 | HM_CHANGED_GUEST_APIC_TPR);
12716 rcStrict = VINF_SUCCESS;
12717 }
12718 }
12719 else
12720 {
12721 /*
12722 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12723 */
12724 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12725 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12726 AssertRCReturn(rc2, rc2);
12727
12728 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12729 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12730
12731 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12732 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12733
12734 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12735 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12736 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12737 }
12738 return VBOXSTRICTRC_TODO(rcStrict);
12739}
12740
12741
12742/**
12743 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12744 * VM-exit.
12745 */
12746HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12747{
12748 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12749 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12750
12751 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12752 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12753 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12754 {
12755 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12756 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12757 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12758 }
12759 else
12760 {
12761 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12762 rcStrict1 = VINF_SUCCESS;
12763 return rcStrict1;
12764 }
12765
12766 RTGCPHYS GCPhys;
12767 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12768 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12769 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12770 AssertRCReturn(rc, rc);
12771
12772 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12773 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12774
12775 RTGCUINT uErrorCode = 0;
12776 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12777 uErrorCode |= X86_TRAP_PF_ID;
12778 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12779 uErrorCode |= X86_TRAP_PF_RW;
12780 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12781 uErrorCode |= X86_TRAP_PF_P;
12782
12783 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12784
12785 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12786 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12787
12788 /* Handle the pagefault trap for the nested shadow table. */
12789 PVM pVM = pVCpu->CTX_SUFF(pVM);
12790 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12791 TRPMResetTrap(pVCpu);
12792
12793 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12794 if ( rcStrict2 == VINF_SUCCESS
12795 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12796 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12797 {
12798 /* Successfully synced our nested page tables. */
12799 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12800 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12801 | HM_CHANGED_GUEST_RSP
12802 | HM_CHANGED_GUEST_RFLAGS);
12803 return VINF_SUCCESS;
12804 }
12805
12806 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12807 return rcStrict2;
12808}
12809
12810/** @} */
12811
12812/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12813/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12814/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12815
12816/** @name VM-exit exception handlers.
12817 * @{
12818 */
12819
12820/**
12821 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12822 */
12823static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12824{
12825 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12826 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12827
12828 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12829 AssertRCReturn(rc, rc);
12830
12831 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12832 {
12833 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12834 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12835
12836 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12837 * provides VM-exit instruction length. If this causes problem later,
12838 * disassemble the instruction like it's done on AMD-V. */
12839 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12840 AssertRCReturn(rc2, rc2);
12841 return rc;
12842 }
12843
12844 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12845 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12846 return rc;
12847}
12848
12849
12850/**
12851 * VM-exit exception handler for \#BP (Breakpoint exception).
12852 */
12853static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12854{
12855 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12856 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12857
12858 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12859 AssertRCReturn(rc, rc);
12860
12861 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12862 if (rc == VINF_EM_RAW_GUEST_TRAP)
12863 {
12864 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12865 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12866 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12867 AssertRCReturn(rc, rc);
12868
12869 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12870 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12871 }
12872
12873 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12874 return rc;
12875}
12876
12877
12878/**
12879 * VM-exit exception handler for \#AC (alignment check exception).
12880 */
12881static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12882{
12883 RT_NOREF_PV(pMixedCtx);
12884 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12885
12886 /*
12887 * Re-inject it. We'll detect any nesting before getting here.
12888 */
12889 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12890 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12891 AssertRCReturn(rc, rc);
12892 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12893
12894 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12895 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12896 return VINF_SUCCESS;
12897}
12898
12899
12900/**
12901 * VM-exit exception handler for \#DB (Debug exception).
12902 */
12903static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12904{
12905 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12906 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12907
12908 /*
12909 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12910 * for processing.
12911 */
12912 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12913
12914 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12915 uint64_t uDR6 = X86_DR6_INIT_VAL;
12916 uDR6 |= ( pVmxTransient->uExitQualification
12917 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12918
12919 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12920 Log6Func(("rc=%Rrc\n", rc));
12921 if (rc == VINF_EM_RAW_GUEST_TRAP)
12922 {
12923 /*
12924 * The exception was for the guest. Update DR6, DR7.GD and
12925 * IA32_DEBUGCTL.LBR before forwarding it.
12926 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12927 */
12928 VMMRZCallRing3Disable(pVCpu);
12929 HM_DISABLE_PREEMPT();
12930
12931 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12932 pMixedCtx->dr[6] |= uDR6;
12933 if (CPUMIsGuestDebugStateActive(pVCpu))
12934 ASMSetDR6(pMixedCtx->dr[6]);
12935
12936 HM_RESTORE_PREEMPT();
12937 VMMRZCallRing3Enable(pVCpu);
12938
12939 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12940 AssertRCReturn(rc, rc);
12941
12942 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12943 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12944
12945 /* Paranoia. */
12946 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12947 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12948
12949 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12950 AssertRCReturn(rc, rc);
12951
12952 /*
12953 * Raise #DB in the guest.
12954 *
12955 * It is important to reflect exactly what the VM-exit gave us (preserving the
12956 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12957 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12958 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12959 *
12960 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12961 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12962 */
12963 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12964 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12965 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12966 AssertRCReturn(rc, rc);
12967 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12968 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12969 return VINF_SUCCESS;
12970 }
12971
12972 /*
12973 * Not a guest trap, must be a hypervisor related debug event then.
12974 * Update DR6 in case someone is interested in it.
12975 */
12976 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12977 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12978 CPUMSetHyperDR6(pVCpu, uDR6);
12979
12980 return rc;
12981}
12982
12983/**
12984 * VM-exit exception handler for \#GP (General-protection exception).
12985 *
12986 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12987 */
12988static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12989{
12990 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12992
12993 int rc;
12994 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12995 { /* likely */ }
12996 else
12997 {
12998#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12999 Assert(pVCpu->hm.s.fUsingDebugLoop);
13000#endif
13001 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
13002 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13003 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13004 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13005 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13006 AssertRCReturn(rc, rc);
13007 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
13008 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
13009 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13010 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13011 return rc;
13012 }
13013
13014 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
13015 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
13016
13017 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
13018 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13019 AssertRCReturn(rc, rc);
13020
13021 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
13022 uint32_t cbOp = 0;
13023 PVM pVM = pVCpu->CTX_SUFF(pVM);
13024 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
13025 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
13026 if (RT_SUCCESS(rc))
13027 {
13028 rc = VINF_SUCCESS;
13029 Assert(cbOp == pDis->cbInstr);
13030 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
13031 switch (pDis->pCurInstr->uOpcode)
13032 {
13033 case OP_CLI:
13034 {
13035 pMixedCtx->eflags.Bits.u1IF = 0;
13036 pMixedCtx->eflags.Bits.u1RF = 0;
13037 pMixedCtx->rip += pDis->cbInstr;
13038 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13039 if ( !fDbgStepping
13040 && pMixedCtx->eflags.Bits.u1TF)
13041 {
13042 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13043 AssertRCReturn(rc, rc);
13044 }
13045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13046 break;
13047 }
13048
13049 case OP_STI:
13050 {
13051 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13052 pMixedCtx->eflags.Bits.u1IF = 1;
13053 pMixedCtx->eflags.Bits.u1RF = 0;
13054 pMixedCtx->rip += pDis->cbInstr;
13055 if (!fOldIF)
13056 {
13057 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13058 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13059 }
13060 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13061 if ( !fDbgStepping
13062 && pMixedCtx->eflags.Bits.u1TF)
13063 {
13064 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13065 AssertRCReturn(rc, rc);
13066 }
13067 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13068 break;
13069 }
13070
13071 case OP_HLT:
13072 {
13073 rc = VINF_EM_HALT;
13074 pMixedCtx->rip += pDis->cbInstr;
13075 pMixedCtx->eflags.Bits.u1RF = 0;
13076 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13077 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13078 break;
13079 }
13080
13081 case OP_POPF:
13082 {
13083 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13084 uint32_t cbParm;
13085 uint32_t uMask;
13086 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13087 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13088 {
13089 cbParm = 4;
13090 uMask = 0xffffffff;
13091 }
13092 else
13093 {
13094 cbParm = 2;
13095 uMask = 0xffff;
13096 }
13097
13098 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13099 RTGCPTR GCPtrStack = 0;
13100 X86EFLAGS Eflags;
13101 Eflags.u32 = 0;
13102 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13103 &GCPtrStack);
13104 if (RT_SUCCESS(rc))
13105 {
13106 Assert(sizeof(Eflags.u32) >= cbParm);
13107 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13108 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13109 }
13110 if (RT_FAILURE(rc))
13111 {
13112 rc = VERR_EM_INTERPRETER;
13113 break;
13114 }
13115 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13116 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13117 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13118 pMixedCtx->esp += cbParm;
13119 pMixedCtx->esp &= uMask;
13120 pMixedCtx->rip += pDis->cbInstr;
13121 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13122 | HM_CHANGED_GUEST_RSP
13123 | HM_CHANGED_GUEST_RFLAGS);
13124 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13125 POPF restores EFLAGS.TF. */
13126 if ( !fDbgStepping
13127 && fGstStepping)
13128 {
13129 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13130 AssertRCReturn(rc, rc);
13131 }
13132 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13133 break;
13134 }
13135
13136 case OP_PUSHF:
13137 {
13138 uint32_t cbParm;
13139 uint32_t uMask;
13140 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13141 {
13142 cbParm = 4;
13143 uMask = 0xffffffff;
13144 }
13145 else
13146 {
13147 cbParm = 2;
13148 uMask = 0xffff;
13149 }
13150
13151 /* Get the stack pointer & push the contents of eflags onto the stack. */
13152 RTGCPTR GCPtrStack = 0;
13153 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13154 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13155 if (RT_FAILURE(rc))
13156 {
13157 rc = VERR_EM_INTERPRETER;
13158 break;
13159 }
13160 X86EFLAGS Eflags = pMixedCtx->eflags;
13161 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13162 Eflags.Bits.u1RF = 0;
13163 Eflags.Bits.u1VM = 0;
13164
13165 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13166 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13167 {
13168 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13169 rc = VERR_EM_INTERPRETER;
13170 break;
13171 }
13172 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13173 pMixedCtx->esp -= cbParm;
13174 pMixedCtx->esp &= uMask;
13175 pMixedCtx->rip += pDis->cbInstr;
13176 pMixedCtx->eflags.Bits.u1RF = 0;
13177 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13178 | HM_CHANGED_GUEST_RSP
13179 | HM_CHANGED_GUEST_RFLAGS);
13180 if ( !fDbgStepping
13181 && pMixedCtx->eflags.Bits.u1TF)
13182 {
13183 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13184 AssertRCReturn(rc, rc);
13185 }
13186 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13187 break;
13188 }
13189
13190 case OP_IRET:
13191 {
13192 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13193 * instruction reference. */
13194 RTGCPTR GCPtrStack = 0;
13195 uint32_t uMask = 0xffff;
13196 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13197 uint16_t aIretFrame[3];
13198 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13199 {
13200 rc = VERR_EM_INTERPRETER;
13201 break;
13202 }
13203 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13204 &GCPtrStack);
13205 if (RT_SUCCESS(rc))
13206 {
13207 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13208 PGMACCESSORIGIN_HM));
13209 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13210 }
13211 if (RT_FAILURE(rc))
13212 {
13213 rc = VERR_EM_INTERPRETER;
13214 break;
13215 }
13216 pMixedCtx->eip = 0;
13217 pMixedCtx->ip = aIretFrame[0];
13218 pMixedCtx->cs.Sel = aIretFrame[1];
13219 pMixedCtx->cs.ValidSel = aIretFrame[1];
13220 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13221 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13222 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13223 pMixedCtx->sp += sizeof(aIretFrame);
13224 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13225 | HM_CHANGED_GUEST_CS
13226 | HM_CHANGED_GUEST_RSP
13227 | HM_CHANGED_GUEST_RFLAGS);
13228 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13229 if ( !fDbgStepping
13230 && fGstStepping)
13231 {
13232 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13233 AssertRCReturn(rc, rc);
13234 }
13235 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13236 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13237 break;
13238 }
13239
13240 case OP_INT:
13241 {
13242 uint16_t uVector = pDis->Param1.uValue & 0xff;
13243 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13244 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13245 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13246 break;
13247 }
13248
13249 case OP_INTO:
13250 {
13251 if (pMixedCtx->eflags.Bits.u1OF)
13252 {
13253 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13254 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13255 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13256 }
13257 else
13258 {
13259 pMixedCtx->eflags.Bits.u1RF = 0;
13260 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13261 }
13262 break;
13263 }
13264
13265 default:
13266 {
13267 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13268 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13269 EMCODETYPE_SUPERVISOR);
13270 rc = VBOXSTRICTRC_VAL(rc2);
13271 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13272 /** @todo We have to set pending-debug exceptions here when the guest is
13273 * single-stepping depending on the instruction that was interpreted. */
13274 Log4Func(("#GP rc=%Rrc\n", rc));
13275 break;
13276 }
13277 }
13278 }
13279 else
13280 rc = VERR_EM_INTERPRETER;
13281
13282 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13283 ("#GP Unexpected rc=%Rrc\n", rc));
13284 return rc;
13285}
13286
13287
13288/**
13289 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13290 * the exception reported in the VMX transient structure back into the VM.
13291 *
13292 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13293 * up-to-date.
13294 */
13295static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13296{
13297 RT_NOREF_PV(pMixedCtx);
13298 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13299#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13300 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13301 ("uVector=%#x u32XcptBitmap=%#X32\n",
13302 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13303#endif
13304
13305 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13306 hmR0VmxCheckExitDueToEventDelivery(). */
13307 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13308 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13309 AssertRCReturn(rc, rc);
13310 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13311
13312#ifdef DEBUG_ramshankar
13313 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13314 | CPUMCTX_EXTRN_RIP);
13315 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13316 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13317#endif
13318
13319 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13320 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13321 return VINF_SUCCESS;
13322}
13323
13324
13325/**
13326 * VM-exit exception handler for \#PF (Page-fault exception).
13327 */
13328static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13329{
13330 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13331 PVM pVM = pVCpu->CTX_SUFF(pVM);
13332 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13333 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13334 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13335 AssertRCReturn(rc, rc);
13336
13337 if (!pVM->hm.s.fNestedPaging)
13338 { /* likely */ }
13339 else
13340 {
13341#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13342 Assert(pVCpu->hm.s.fUsingDebugLoop);
13343#endif
13344 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13345 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13346 {
13347 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13348 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13349 }
13350 else
13351 {
13352 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13353 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13354 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13355 }
13356 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13357 return rc;
13358 }
13359
13360 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13361 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13362 if (pVmxTransient->fVectoringPF)
13363 {
13364 Assert(pVCpu->hm.s.Event.fPending);
13365 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13366 }
13367
13368 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13369 AssertRCReturn(rc, rc);
13370
13371 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13372 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13373
13374 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13375 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13376 (RTGCPTR)pVmxTransient->uExitQualification);
13377
13378 Log4Func(("#PF: rc=%Rrc\n", rc));
13379 if (rc == VINF_SUCCESS)
13380 {
13381 /*
13382 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13383 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13384 */
13385 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13386 TRPMResetTrap(pVCpu);
13387 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13388 return rc;
13389 }
13390
13391 if (rc == VINF_EM_RAW_GUEST_TRAP)
13392 {
13393 if (!pVmxTransient->fVectoringDoublePF)
13394 {
13395 /* It's a guest page fault and needs to be reflected to the guest. */
13396 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13397 TRPMResetTrap(pVCpu);
13398 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13399 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13400 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13401 }
13402 else
13403 {
13404 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13405 TRPMResetTrap(pVCpu);
13406 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13407 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13408 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13409 }
13410
13411 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13412 return VINF_SUCCESS;
13413 }
13414
13415 TRPMResetTrap(pVCpu);
13416 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13417 return rc;
13418}
13419
13420/** @} */
13421
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette