VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72848

Last change on this file since 72848 was 72848, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: bugref:9193 Always import FS, GS (for base) on MSR read/write VM-exits. Export strictly only those
segment registers that were marked as changed and not all of them as we risk overwriting stale values from
the guest-CPU into the VMCS values.

  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 574.4 KB
Line 
1/* $Id: HMVMXR0.cpp 72848 2018-07-04 05:01:54Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for importing segment registers to the VMCS from the guest-CPU context. */
173#ifdef VMX_USE_CACHED_VMCS_ACCESSES
174# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
175 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
176 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
177#else
178# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
179 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
180 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
181#endif
182
183/** Macro for exporting segment registers to the VMCS from the guest-CPU context. */
184# define HMVMX_EXPORT_SREG(Sel, a_pCtxSelReg) \
185 hmR0VmxExportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
186 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
187
188
189/*********************************************************************************************************************************
190* Structures and Typedefs *
191*********************************************************************************************************************************/
192/**
193 * VMX transient state.
194 *
195 * A state structure for holding miscellaneous information across
196 * VMX non-root operation and restored after the transition.
197 */
198typedef struct VMXTRANSIENT
199{
200 /** The host's rflags/eflags. */
201 RTCCUINTREG fEFlags;
202#if HC_ARCH_BITS == 32
203 uint32_t u32Alignment0;
204#endif
205 /** The guest's TPR value used for TPR shadowing. */
206 uint8_t u8GuestTpr;
207 /** Alignment. */
208 uint8_t abAlignment0[7];
209
210 /** The basic VM-exit reason. */
211 uint16_t uExitReason;
212 /** Alignment. */
213 uint16_t u16Alignment0;
214 /** The VM-exit interruption error code. */
215 uint32_t uExitIntErrorCode;
216 /** The VM-exit exit code qualification. */
217 uint64_t uExitQualification;
218
219 /** The VM-exit interruption-information field. */
220 uint32_t uExitIntInfo;
221 /** The VM-exit instruction-length field. */
222 uint32_t cbInstr;
223 /** The VM-exit instruction-information field. */
224 union
225 {
226 /** Plain unsigned int representation. */
227 uint32_t u;
228 /** INS and OUTS information. */
229 struct
230 {
231 uint32_t u7Reserved0 : 7;
232 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
233 uint32_t u3AddrSize : 3;
234 uint32_t u5Reserved1 : 5;
235 /** The segment register (X86_SREG_XXX). */
236 uint32_t iSegReg : 3;
237 uint32_t uReserved2 : 14;
238 } StrIo;
239 /** INVEPT, INVVPID, INVPCID information. */
240 struct
241 {
242 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
243 uint32_t u2Scaling : 2;
244 uint32_t u5Reserved0 : 5;
245 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
246 uint32_t u3AddrSize : 3;
247 uint32_t u1Reserved0 : 1;
248 uint32_t u4Reserved0 : 4;
249 /** The segment register (X86_SREG_XXX). */
250 uint32_t iSegReg : 3;
251 /** The index register (X86_GREG_XXX). */
252 uint32_t iIdxReg : 4;
253 /** Set if index register is invalid. */
254 uint32_t fIdxRegValid : 1;
255 /** The base register (X86_GREG_XXX). */
256 uint32_t iBaseReg : 4;
257 /** Set if base register is invalid. */
258 uint32_t fBaseRegValid : 1;
259 /** Register 2 (X86_GREG_XXX). */
260 uint32_t iReg2 : 4;
261 } Inv;
262 } ExitInstrInfo;
263 /** Whether the VM-entry failed or not. */
264 bool fVMEntryFailed;
265 /** Alignment. */
266 uint8_t abAlignment1[3];
267
268 /** The VM-entry interruption-information field. */
269 uint32_t uEntryIntInfo;
270 /** The VM-entry exception error code field. */
271 uint32_t uEntryXcptErrorCode;
272 /** The VM-entry instruction length field. */
273 uint32_t cbEntryInstr;
274
275 /** IDT-vectoring information field. */
276 uint32_t uIdtVectoringInfo;
277 /** IDT-vectoring error code. */
278 uint32_t uIdtVectoringErrorCode;
279
280 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
281 uint32_t fVmcsFieldsRead;
282
283 /** Whether the guest debug state was active at the time of VM-exit. */
284 bool fWasGuestDebugStateActive;
285 /** Whether the hyper debug state was active at the time of VM-exit. */
286 bool fWasHyperDebugStateActive;
287 /** Whether TSC-offsetting should be setup before VM-entry. */
288 bool fUpdateTscOffsettingAndPreemptTimer;
289 /** Whether the VM-exit was caused by a page-fault during delivery of a
290 * contributory exception or a page-fault. */
291 bool fVectoringDoublePF;
292 /** Whether the VM-exit was caused by a page-fault during delivery of an
293 * external interrupt or NMI. */
294 bool fVectoringPF;
295} VMXTRANSIENT;
296AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
297AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
298AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
299AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
300AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
301/** Pointer to VMX transient state. */
302typedef VMXTRANSIENT *PVMXTRANSIENT;
303
304
305/**
306 * MSR-bitmap read permissions.
307 */
308typedef enum VMXMSREXITREAD
309{
310 /** Reading this MSR causes a VM-exit. */
311 VMXMSREXIT_INTERCEPT_READ = 0xb,
312 /** Reading this MSR does not cause a VM-exit. */
313 VMXMSREXIT_PASSTHRU_READ
314} VMXMSREXITREAD;
315/** Pointer to MSR-bitmap read permissions. */
316typedef VMXMSREXITREAD* PVMXMSREXITREAD;
317
318/**
319 * MSR-bitmap write permissions.
320 */
321typedef enum VMXMSREXITWRITE
322{
323 /** Writing to this MSR causes a VM-exit. */
324 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
325 /** Writing to this MSR does not cause a VM-exit. */
326 VMXMSREXIT_PASSTHRU_WRITE
327} VMXMSREXITWRITE;
328/** Pointer to MSR-bitmap write permissions. */
329typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
330
331
332/**
333 * VMX VM-exit handler.
334 *
335 * @returns Strict VBox status code (i.e. informational status codes too).
336 * @param pVCpu The cross context virtual CPU structure.
337 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
338 * out-of-sync. Make sure to update the required
339 * fields before using them.
340 * @param pVmxTransient Pointer to the VMX-transient structure.
341 */
342#ifndef HMVMX_USE_FUNCTION_TABLE
343typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
344#else
345typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
346/** Pointer to VM-exit handler. */
347typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
348#endif
349
350/**
351 * VMX VM-exit handler, non-strict status code.
352 *
353 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
354 *
355 * @returns VBox status code, no informational status code returned.
356 * @param pVCpu The cross context virtual CPU structure.
357 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
358 * out-of-sync. Make sure to update the required
359 * fields before using them.
360 * @param pVmxTransient Pointer to the VMX-transient structure.
361 *
362 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
363 * use of that status code will be replaced with VINF_EM_SOMETHING
364 * later when switching over to IEM.
365 */
366#ifndef HMVMX_USE_FUNCTION_TABLE
367typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
368#else
369typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
370#endif
371
372
373/*********************************************************************************************************************************
374* Internal Functions *
375*********************************************************************************************************************************/
376static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
377static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
378static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
379static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
380static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
381 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
382#if HC_ARCH_BITS == 32
383static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
384#endif
385#ifndef HMVMX_USE_FUNCTION_TABLE
386DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
387# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
388# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
389#else
390# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
391# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
392#endif
393
394
395/** @name VM-exit handlers.
396 * @{
397 */
398static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
399static FNVMXEXITHANDLER hmR0VmxExitExtInt;
400static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
403static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
404static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
405static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
406static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
407static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
408static FNVMXEXITHANDLER hmR0VmxExitCpuid;
409static FNVMXEXITHANDLER hmR0VmxExitGetsec;
410static FNVMXEXITHANDLER hmR0VmxExitHlt;
411static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
412static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
413static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
414static FNVMXEXITHANDLER hmR0VmxExitVmcall;
415static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
416static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
417static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
418static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
419static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
420static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
421static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
422static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
423static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
424static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
425static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
426static FNVMXEXITHANDLER hmR0VmxExitMwait;
427static FNVMXEXITHANDLER hmR0VmxExitMtf;
428static FNVMXEXITHANDLER hmR0VmxExitMonitor;
429static FNVMXEXITHANDLER hmR0VmxExitPause;
430static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
431static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
432static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
433static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
434static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
435static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
436static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
437static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
438static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
439static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
440static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
441static FNVMXEXITHANDLER hmR0VmxExitRdrand;
442static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
443/** @} */
444
445static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
449static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
450static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
451static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
452static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
453
454
455/*********************************************************************************************************************************
456* Global Variables *
457*********************************************************************************************************************************/
458#ifdef HMVMX_USE_FUNCTION_TABLE
459
460/**
461 * VMX_EXIT dispatch table.
462 */
463static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
464{
465 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
466 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
467 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
468 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
469 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
470 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
471 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
472 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
473 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
474 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
475 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
476 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
477 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
478 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
479 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
480 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
481 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
482 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
483 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
484 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
485 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
486 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
487 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
488 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
489 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
490 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
491 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
492 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
493 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
494 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
495 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
496 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
497 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
498 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
499 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
500 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
501 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
502 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
503 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
504 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
505 /* 40 UNDEFINED */ hmR0VmxExitPause,
506 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
507 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
508 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
509 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
510 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
511 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
512 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
513 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
514 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
515 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
516 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
517 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
518 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
519 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
520 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
521 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
522 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
523 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
524 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
525 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
526 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
527 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
528 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
529 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
530};
531#endif /* HMVMX_USE_FUNCTION_TABLE */
532
533#ifdef VBOX_STRICT
534static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
535{
536 /* 0 */ "(Not Used)",
537 /* 1 */ "VMCALL executed in VMX root operation.",
538 /* 2 */ "VMCLEAR with invalid physical address.",
539 /* 3 */ "VMCLEAR with VMXON pointer.",
540 /* 4 */ "VMLAUNCH with non-clear VMCS.",
541 /* 5 */ "VMRESUME with non-launched VMCS.",
542 /* 6 */ "VMRESUME after VMXOFF",
543 /* 7 */ "VM-entry with invalid control fields.",
544 /* 8 */ "VM-entry with invalid host state fields.",
545 /* 9 */ "VMPTRLD with invalid physical address.",
546 /* 10 */ "VMPTRLD with VMXON pointer.",
547 /* 11 */ "VMPTRLD with incorrect revision identifier.",
548 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
549 /* 13 */ "VMWRITE to read-only VMCS component.",
550 /* 14 */ "(Not Used)",
551 /* 15 */ "VMXON executed in VMX root operation.",
552 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
553 /* 17 */ "VM-entry with non-launched executing VMCS.",
554 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
555 /* 19 */ "VMCALL with non-clear VMCS.",
556 /* 20 */ "VMCALL with invalid VM-exit control fields.",
557 /* 21 */ "(Not Used)",
558 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
559 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
560 /* 24 */ "VMCALL with invalid SMM-monitor features.",
561 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
562 /* 26 */ "VM-entry with events blocked by MOV SS.",
563 /* 27 */ "(Not Used)",
564 /* 28 */ "Invalid operand to INVEPT/INVVPID."
565};
566#endif /* VBOX_STRICT */
567
568
569
570/**
571 * Updates the VM's last error record.
572 *
573 * If there was a VMX instruction error, reads the error data from the VMCS and
574 * updates VCPU's last error record as well.
575 *
576 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
577 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
578 * VERR_VMX_INVALID_VMCS_FIELD.
579 * @param rc The error code.
580 */
581static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
582{
583 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
584 || rc == VERR_VMX_UNABLE_TO_START_VM)
585 {
586 AssertPtrReturnVoid(pVCpu);
587 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
588 }
589 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
590}
591
592
593/**
594 * Reads the VM-entry interruption-information field from the VMCS into the VMX
595 * transient structure.
596 *
597 * @returns VBox status code.
598 * @param pVmxTransient Pointer to the VMX transient structure.
599 *
600 * @remarks No-long-jump zone!!!
601 */
602DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
603{
604 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
605 AssertRCReturn(rc, rc);
606 return VINF_SUCCESS;
607}
608
609#ifdef VBOX_STRICT
610/**
611 * Reads the VM-entry exception error code field from the VMCS into
612 * the VMX transient structure.
613 *
614 * @returns VBox status code.
615 * @param pVmxTransient Pointer to the VMX transient structure.
616 *
617 * @remarks No-long-jump zone!!!
618 */
619DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
620{
621 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
622 AssertRCReturn(rc, rc);
623 return VINF_SUCCESS;
624}
625
626
627/**
628 * Reads the VM-entry exception error code field from the VMCS into
629 * the VMX transient structure.
630 *
631 * @returns VBox status code.
632 * @param pVmxTransient Pointer to the VMX transient structure.
633 *
634 * @remarks No-long-jump zone!!!
635 */
636DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
637{
638 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
639 AssertRCReturn(rc, rc);
640 return VINF_SUCCESS;
641}
642#endif /* VBOX_STRICT */
643
644
645/**
646 * Reads the VM-exit interruption-information field from the VMCS into the VMX
647 * transient structure.
648 *
649 * @returns VBox status code.
650 * @param pVmxTransient Pointer to the VMX transient structure.
651 */
652DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
653{
654 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
655 {
656 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
657 AssertRCReturn(rc,rc);
658 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
659 }
660 return VINF_SUCCESS;
661}
662
663
664/**
665 * Reads the VM-exit interruption error code from the VMCS into the VMX
666 * transient structure.
667 *
668 * @returns VBox status code.
669 * @param pVmxTransient Pointer to the VMX transient structure.
670 */
671DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
672{
673 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
674 {
675 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
676 AssertRCReturn(rc, rc);
677 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
678 }
679 return VINF_SUCCESS;
680}
681
682
683/**
684 * Reads the VM-exit instruction length field from the VMCS into the VMX
685 * transient structure.
686 *
687 * @returns VBox status code.
688 * @param pVmxTransient Pointer to the VMX transient structure.
689 */
690DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
691{
692 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
693 {
694 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
695 AssertRCReturn(rc, rc);
696 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
697 }
698 return VINF_SUCCESS;
699}
700
701
702/**
703 * Reads the VM-exit instruction-information field from the VMCS into
704 * the VMX transient structure.
705 *
706 * @returns VBox status code.
707 * @param pVmxTransient Pointer to the VMX transient structure.
708 */
709DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
710{
711 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
712 {
713 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
714 AssertRCReturn(rc, rc);
715 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
716 }
717 return VINF_SUCCESS;
718}
719
720
721/**
722 * Reads the exit code qualification from the VMCS into the VMX transient
723 * structure.
724 *
725 * @returns VBox status code.
726 * @param pVCpu The cross context virtual CPU structure of the
727 * calling EMT. (Required for the VMCS cache case.)
728 * @param pVmxTransient Pointer to the VMX transient structure.
729 */
730DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
731{
732 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
733 {
734 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
735 AssertRCReturn(rc, rc);
736 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
737 }
738 return VINF_SUCCESS;
739}
740
741
742/**
743 * Reads the IDT-vectoring information field from the VMCS into the VMX
744 * transient structure.
745 *
746 * @returns VBox status code.
747 * @param pVmxTransient Pointer to the VMX transient structure.
748 *
749 * @remarks No-long-jump zone!!!
750 */
751DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
752{
753 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
754 {
755 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
756 AssertRCReturn(rc, rc);
757 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
758 }
759 return VINF_SUCCESS;
760}
761
762
763/**
764 * Reads the IDT-vectoring error code from the VMCS into the VMX
765 * transient structure.
766 *
767 * @returns VBox status code.
768 * @param pVmxTransient Pointer to the VMX transient structure.
769 */
770DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
771{
772 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
773 {
774 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
775 AssertRCReturn(rc, rc);
776 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
777 }
778 return VINF_SUCCESS;
779}
780
781
782/**
783 * Enters VMX root mode operation on the current CPU.
784 *
785 * @returns VBox status code.
786 * @param pVM The cross context VM structure. Can be
787 * NULL, after a resume.
788 * @param HCPhysCpuPage Physical address of the VMXON region.
789 * @param pvCpuPage Pointer to the VMXON region.
790 */
791static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
792{
793 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
794 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
795 Assert(pvCpuPage);
796 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
797
798 if (pVM)
799 {
800 /* Write the VMCS revision dword to the VMXON region. */
801 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
802 }
803
804 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
805 RTCCUINTREG fEFlags = ASMIntDisableFlags();
806
807 /* Enable the VMX bit in CR4 if necessary. */
808 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
809
810 /* Enter VMX root mode. */
811 int rc = VMXEnable(HCPhysCpuPage);
812 if (RT_FAILURE(rc))
813 {
814 if (!(uOldCr4 & X86_CR4_VMXE))
815 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
816
817 if (pVM)
818 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
819 }
820
821 /* Restore interrupts. */
822 ASMSetFlags(fEFlags);
823 return rc;
824}
825
826
827/**
828 * Exits VMX root mode operation on the current CPU.
829 *
830 * @returns VBox status code.
831 */
832static int hmR0VmxLeaveRootMode(void)
833{
834 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
835
836 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
837 RTCCUINTREG fEFlags = ASMIntDisableFlags();
838
839 /* If we're for some reason not in VMX root mode, then don't leave it. */
840 RTCCUINTREG uHostCR4 = ASMGetCR4();
841
842 int rc;
843 if (uHostCR4 & X86_CR4_VMXE)
844 {
845 /* Exit VMX root mode and clear the VMX bit in CR4. */
846 VMXDisable();
847 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
848 rc = VINF_SUCCESS;
849 }
850 else
851 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
852
853 /* Restore interrupts. */
854 ASMSetFlags(fEFlags);
855 return rc;
856}
857
858
859/**
860 * Allocates and maps one physically contiguous page. The allocated page is
861 * zero'd out. (Used by various VT-x structures).
862 *
863 * @returns IPRT status code.
864 * @param pMemObj Pointer to the ring-0 memory object.
865 * @param ppVirt Where to store the virtual address of the
866 * allocation.
867 * @param pHCPhys Where to store the physical address of the
868 * allocation.
869 */
870static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
871{
872 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
873 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
874 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
875
876 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
877 if (RT_FAILURE(rc))
878 return rc;
879 *ppVirt = RTR0MemObjAddress(*pMemObj);
880 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
881 ASMMemZero32(*ppVirt, PAGE_SIZE);
882 return VINF_SUCCESS;
883}
884
885
886/**
887 * Frees and unmaps an allocated physical page.
888 *
889 * @param pMemObj Pointer to the ring-0 memory object.
890 * @param ppVirt Where to re-initialize the virtual address of
891 * allocation as 0.
892 * @param pHCPhys Where to re-initialize the physical address of the
893 * allocation as 0.
894 */
895static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
896{
897 AssertPtr(pMemObj);
898 AssertPtr(ppVirt);
899 AssertPtr(pHCPhys);
900 if (*pMemObj != NIL_RTR0MEMOBJ)
901 {
902 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
903 AssertRC(rc);
904 *pMemObj = NIL_RTR0MEMOBJ;
905 *ppVirt = 0;
906 *pHCPhys = 0;
907 }
908}
909
910
911/**
912 * Worker function to free VT-x related structures.
913 *
914 * @returns IPRT status code.
915 * @param pVM The cross context VM structure.
916 */
917static void hmR0VmxStructsFree(PVM pVM)
918{
919 for (VMCPUID i = 0; i < pVM->cCpus; i++)
920 {
921 PVMCPU pVCpu = &pVM->aCpus[i];
922 AssertPtr(pVCpu);
923
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
925 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
926
927 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
928 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
929
930 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
931 }
932
933 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
934#ifdef VBOX_WITH_CRASHDUMP_MAGIC
935 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
936#endif
937}
938
939
940/**
941 * Worker function to allocate VT-x related VM structures.
942 *
943 * @returns IPRT status code.
944 * @param pVM The cross context VM structure.
945 */
946static int hmR0VmxStructsAlloc(PVM pVM)
947{
948 /*
949 * Initialize members up-front so we can cleanup properly on allocation failure.
950 */
951#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
952 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
953 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
954 pVM->hm.s.vmx.HCPhys##a_Name = 0;
955
956#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
957 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
958 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
959 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
960
961#ifdef VBOX_WITH_CRASHDUMP_MAGIC
962 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
963#endif
964 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
965
966 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
967 for (VMCPUID i = 0; i < pVM->cCpus; i++)
968 {
969 PVMCPU pVCpu = &pVM->aCpus[i];
970 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
971 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
972 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
973 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
974 }
975#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
976#undef VMXLOCAL_INIT_VM_MEMOBJ
977
978 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
979 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
980 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
981 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
982
983 /*
984 * Allocate all the VT-x structures.
985 */
986 int rc = VINF_SUCCESS;
987#ifdef VBOX_WITH_CRASHDUMP_MAGIC
988 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
989 if (RT_FAILURE(rc))
990 goto cleanup;
991 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
992 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
993#endif
994
995 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
996 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
997 {
998 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
999 &pVM->hm.s.vmx.HCPhysApicAccess);
1000 if (RT_FAILURE(rc))
1001 goto cleanup;
1002 }
1003
1004 /*
1005 * Initialize per-VCPU VT-x structures.
1006 */
1007 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1008 {
1009 PVMCPU pVCpu = &pVM->aCpus[i];
1010 AssertPtr(pVCpu);
1011
1012 /* Allocate the VM control structure (VMCS). */
1013 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1014 if (RT_FAILURE(rc))
1015 goto cleanup;
1016
1017 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1018 if ( PDMHasApic(pVM)
1019 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1020 {
1021 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1022 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1023 if (RT_FAILURE(rc))
1024 goto cleanup;
1025 }
1026
1027 /*
1028 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1029 * transparent accesses of specific MSRs.
1030 *
1031 * If the condition for enabling MSR bitmaps changes here, don't forget to
1032 * update HMAreMsrBitmapsAvailable().
1033 */
1034 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1035 {
1036 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1037 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1038 if (RT_FAILURE(rc))
1039 goto cleanup;
1040 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1041 }
1042
1043 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1044 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1045 if (RT_FAILURE(rc))
1046 goto cleanup;
1047
1048 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1049 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1050 if (RT_FAILURE(rc))
1051 goto cleanup;
1052 }
1053
1054 return VINF_SUCCESS;
1055
1056cleanup:
1057 hmR0VmxStructsFree(pVM);
1058 return rc;
1059}
1060
1061
1062/**
1063 * Does global VT-x initialization (called during module initialization).
1064 *
1065 * @returns VBox status code.
1066 */
1067VMMR0DECL(int) VMXR0GlobalInit(void)
1068{
1069#ifdef HMVMX_USE_FUNCTION_TABLE
1070 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1071# ifdef VBOX_STRICT
1072 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1073 Assert(g_apfnVMExitHandlers[i]);
1074# endif
1075#endif
1076 return VINF_SUCCESS;
1077}
1078
1079
1080/**
1081 * Does global VT-x termination (called during module termination).
1082 */
1083VMMR0DECL(void) VMXR0GlobalTerm()
1084{
1085 /* Nothing to do currently. */
1086}
1087
1088
1089/**
1090 * Sets up and activates VT-x on the current CPU.
1091 *
1092 * @returns VBox status code.
1093 * @param pHostCpu Pointer to the global CPU info struct.
1094 * @param pVM The cross context VM structure. Can be
1095 * NULL after a host resume operation.
1096 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1097 * fEnabledByHost is @c true).
1098 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1099 * @a fEnabledByHost is @c true).
1100 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1101 * enable VT-x on the host.
1102 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1103 */
1104VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1105 void *pvMsrs)
1106{
1107 Assert(pHostCpu);
1108 Assert(pvMsrs);
1109 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1110
1111 /* Enable VT-x if it's not already enabled by the host. */
1112 if (!fEnabledByHost)
1113 {
1114 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1115 if (RT_FAILURE(rc))
1116 return rc;
1117 }
1118
1119 /*
1120 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1121 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1122 * invalidated when flushing by VPID.
1123 */
1124 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1125 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1126 {
1127 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1128 pHostCpu->fFlushAsidBeforeUse = false;
1129 }
1130 else
1131 pHostCpu->fFlushAsidBeforeUse = true;
1132
1133 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1134 ++pHostCpu->cTlbFlushes;
1135
1136 return VINF_SUCCESS;
1137}
1138
1139
1140/**
1141 * Deactivates VT-x on the current CPU.
1142 *
1143 * @returns VBox status code.
1144 * @param pHostCpu Pointer to the global CPU info struct.
1145 * @param pvCpuPage Pointer to the VMXON region.
1146 * @param HCPhysCpuPage Physical address of the VMXON region.
1147 *
1148 * @remarks This function should never be called when SUPR0EnableVTx() or
1149 * similar was used to enable VT-x on the host.
1150 */
1151VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1152{
1153 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1154
1155 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1156 return hmR0VmxLeaveRootMode();
1157}
1158
1159
1160/**
1161 * Sets the permission bits for the specified MSR in the MSR bitmap.
1162 *
1163 * @param pVCpu The cross context virtual CPU structure.
1164 * @param uMsr The MSR value.
1165 * @param enmRead Whether reading this MSR causes a VM-exit.
1166 * @param enmWrite Whether writing this MSR causes a VM-exit.
1167 */
1168static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1169{
1170 int32_t iBit;
1171 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1172
1173 /*
1174 * Layout:
1175 * 0x000 - 0x3ff - Low MSR read bits
1176 * 0x400 - 0x7ff - High MSR read bits
1177 * 0x800 - 0xbff - Low MSR write bits
1178 * 0xc00 - 0xfff - High MSR write bits
1179 */
1180 if (uMsr <= 0x00001FFF)
1181 iBit = uMsr;
1182 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1183 {
1184 iBit = uMsr - UINT32_C(0xC0000000);
1185 pbMsrBitmap += 0x400;
1186 }
1187 else
1188 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1189
1190 Assert(iBit <= 0x1fff);
1191 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1192 ASMBitSet(pbMsrBitmap, iBit);
1193 else
1194 ASMBitClear(pbMsrBitmap, iBit);
1195
1196 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1197 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1198 else
1199 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1200}
1201
1202
1203#ifdef VBOX_STRICT
1204/**
1205 * Gets the permission bits for the specified MSR in the MSR bitmap.
1206 *
1207 * @returns VBox status code.
1208 * @retval VINF_SUCCESS if the specified MSR is found.
1209 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1210 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1211 *
1212 * @param pVCpu The cross context virtual CPU structure.
1213 * @param uMsr The MSR.
1214 * @param penmRead Where to store the read permissions.
1215 * @param penmWrite Where to store the write permissions.
1216 */
1217static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1218{
1219 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1220 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1221 int32_t iBit;
1222 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1223
1224 /* See hmR0VmxSetMsrPermission() for the layout. */
1225 if (uMsr <= 0x00001FFF)
1226 iBit = uMsr;
1227 else if ( uMsr >= 0xC0000000
1228 && uMsr <= 0xC0001FFF)
1229 {
1230 iBit = (uMsr - 0xC0000000);
1231 pbMsrBitmap += 0x400;
1232 }
1233 else
1234 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1235
1236 Assert(iBit <= 0x1fff);
1237 if (ASMBitTest(pbMsrBitmap, iBit))
1238 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1239 else
1240 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1241
1242 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1243 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1244 else
1245 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1246 return VINF_SUCCESS;
1247}
1248#endif /* VBOX_STRICT */
1249
1250
1251/**
1252 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1253 * area.
1254 *
1255 * @returns VBox status code.
1256 * @param pVCpu The cross context virtual CPU structure.
1257 * @param cMsrs The number of MSRs.
1258 */
1259static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1260{
1261 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1262 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1263 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1264 {
1265 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1266 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1267 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1268 }
1269
1270 /* Update number of guest MSRs to load/store across the world-switch. */
1271 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1272 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1273
1274 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1275 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1276 AssertRCReturn(rc, rc);
1277
1278 /* Update the VCPU's copy of the MSR count. */
1279 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1280
1281 return VINF_SUCCESS;
1282}
1283
1284
1285/**
1286 * Adds a new (or updates the value of an existing) guest/host MSR
1287 * pair to be swapped during the world-switch as part of the
1288 * auto-load/store MSR area in the VMCS.
1289 *
1290 * @returns VBox status code.
1291 * @param pVCpu The cross context virtual CPU structure.
1292 * @param uMsr The MSR.
1293 * @param uGuestMsrValue Value of the guest MSR.
1294 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1295 * necessary.
1296 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1297 * its value was updated. Optional, can be NULL.
1298 */
1299static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1300 bool *pfAddedAndUpdated)
1301{
1302 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1303 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1304 uint32_t i;
1305 for (i = 0; i < cMsrs; i++)
1306 {
1307 if (pGuestMsr->u32Msr == uMsr)
1308 break;
1309 pGuestMsr++;
1310 }
1311
1312 bool fAdded = false;
1313 if (i == cMsrs)
1314 {
1315 ++cMsrs;
1316 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1317 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1318
1319 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1320 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1321 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1322
1323 fAdded = true;
1324 }
1325
1326 /* Update the MSR values in the auto-load/store MSR area. */
1327 pGuestMsr->u32Msr = uMsr;
1328 pGuestMsr->u64Value = uGuestMsrValue;
1329
1330 /* Create/update the MSR slot in the host MSR area. */
1331 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1332 pHostMsr += i;
1333 pHostMsr->u32Msr = uMsr;
1334
1335 /*
1336 * Update the host MSR only when requested by the caller AND when we're
1337 * adding it to the auto-load/store area. Otherwise, it would have been
1338 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1339 */
1340 bool fUpdatedMsrValue = false;
1341 if ( fAdded
1342 && fUpdateHostMsr)
1343 {
1344 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1345 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1346 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1347 fUpdatedMsrValue = true;
1348 }
1349
1350 if (pfAddedAndUpdated)
1351 *pfAddedAndUpdated = fUpdatedMsrValue;
1352 return VINF_SUCCESS;
1353}
1354
1355
1356/**
1357 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1358 * auto-load/store MSR area in the VMCS.
1359 *
1360 * @returns VBox status code.
1361 * @param pVCpu The cross context virtual CPU structure.
1362 * @param uMsr The MSR.
1363 */
1364static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1365{
1366 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1367 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1368 for (uint32_t i = 0; i < cMsrs; i++)
1369 {
1370 /* Find the MSR. */
1371 if (pGuestMsr->u32Msr == uMsr)
1372 {
1373 /* If it's the last MSR, simply reduce the count. */
1374 if (i == cMsrs - 1)
1375 {
1376 --cMsrs;
1377 break;
1378 }
1379
1380 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1381 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1382 pLastGuestMsr += cMsrs - 1;
1383 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1384 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1385
1386 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1387 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1388 pLastHostMsr += cMsrs - 1;
1389 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1390 pHostMsr->u64Value = pLastHostMsr->u64Value;
1391 --cMsrs;
1392 break;
1393 }
1394 pGuestMsr++;
1395 }
1396
1397 /* Update the VMCS if the count changed (meaning the MSR was found). */
1398 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1399 {
1400 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1401 AssertRCReturn(rc, rc);
1402
1403 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1404 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1405 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1406
1407 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1408 return VINF_SUCCESS;
1409 }
1410
1411 return VERR_NOT_FOUND;
1412}
1413
1414
1415/**
1416 * Checks if the specified guest MSR is part of the auto-load/store area in
1417 * the VMCS.
1418 *
1419 * @returns true if found, false otherwise.
1420 * @param pVCpu The cross context virtual CPU structure.
1421 * @param uMsr The MSR to find.
1422 */
1423static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1424{
1425 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1426 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1427
1428 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1429 {
1430 if (pGuestMsr->u32Msr == uMsr)
1431 return true;
1432 }
1433 return false;
1434}
1435
1436
1437/**
1438 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1439 *
1440 * @param pVCpu The cross context virtual CPU structure.
1441 *
1442 * @remarks No-long-jump zone!!!
1443 */
1444static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1445{
1446 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1447 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1448 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1449 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1450
1451 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1452 {
1453 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1454
1455 /*
1456 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1457 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1458 */
1459 if (pHostMsr->u32Msr == MSR_K6_EFER)
1460 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1461 else
1462 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1463 }
1464
1465 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1466}
1467
1468
1469/**
1470 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1471 * perform lazy restoration of the host MSRs while leaving VT-x.
1472 *
1473 * @param pVCpu The cross context virtual CPU structure.
1474 *
1475 * @remarks No-long-jump zone!!!
1476 */
1477static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1478{
1479 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1480
1481 /*
1482 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1483 */
1484 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1485 {
1486 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1487#if HC_ARCH_BITS == 64
1488 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1489 {
1490 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1491 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1492 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1493 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1494 }
1495#endif
1496 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1497 }
1498}
1499
1500
1501/**
1502 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1503 * lazily while leaving VT-x.
1504 *
1505 * @returns true if it does, false otherwise.
1506 * @param pVCpu The cross context virtual CPU structure.
1507 * @param uMsr The MSR to check.
1508 */
1509static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1510{
1511 NOREF(pVCpu);
1512#if HC_ARCH_BITS == 64
1513 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1514 {
1515 switch (uMsr)
1516 {
1517 case MSR_K8_LSTAR:
1518 case MSR_K6_STAR:
1519 case MSR_K8_SF_MASK:
1520 case MSR_K8_KERNEL_GS_BASE:
1521 return true;
1522 }
1523 }
1524#else
1525 RT_NOREF(pVCpu, uMsr);
1526#endif
1527 return false;
1528}
1529
1530
1531/**
1532 * Loads a set of guests MSRs to allow read/passthru to the guest.
1533 *
1534 * The name of this function is slightly confusing. This function does NOT
1535 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1536 * common prefix for functions dealing with "lazy restoration" of the shared
1537 * MSRs.
1538 *
1539 * @param pVCpu The cross context virtual CPU structure.
1540 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1541 * out-of-sync. Make sure to update the required fields
1542 * before using them.
1543 *
1544 * @remarks No-long-jump zone!!!
1545 */
1546static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1547{
1548 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1549 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1550
1551 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1552#if HC_ARCH_BITS == 64
1553 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1554 {
1555 /*
1556 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1557 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1558 * we can skip a few MSR writes.
1559 *
1560 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1561 * guest MSR values in the guest-CPU context might be different to what's currently
1562 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1563 * CPU, see @bugref{8728}.
1564 */
1565 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1566 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1567 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1568 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1569 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1570 {
1571#ifdef VBOX_STRICT
1572 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1573 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1574 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1575 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1576#endif
1577 }
1578 else
1579 {
1580 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1581 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1582 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1583 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1584 }
1585 }
1586#else
1587 RT_NOREF(pMixedCtx);
1588#endif
1589 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1590}
1591
1592
1593/**
1594 * Performs lazy restoration of the set of host MSRs if they were previously
1595 * loaded with guest MSR values.
1596 *
1597 * @param pVCpu The cross context virtual CPU structure.
1598 *
1599 * @remarks No-long-jump zone!!!
1600 * @remarks The guest MSRs should have been saved back into the guest-CPU
1601 * context by hmR0VmxImportGuestState()!!!
1602 */
1603static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1604{
1605 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1606 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1607
1608 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1609 {
1610 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1611#if HC_ARCH_BITS == 64
1612 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1613 {
1614 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1615 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1616 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1617 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1618 }
1619#endif
1620 }
1621 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1622}
1623
1624
1625/**
1626 * Verifies that our cached values of the VMCS controls are all
1627 * consistent with what's actually present in the VMCS.
1628 *
1629 * @returns VBox status code.
1630 * @param pVCpu The cross context virtual CPU structure.
1631 */
1632static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1633{
1634 uint32_t u32Val;
1635 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1636 AssertRCReturn(rc, rc);
1637 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1638 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1639
1640 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1641 AssertRCReturn(rc, rc);
1642 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1643 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1644
1645 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1646 AssertRCReturn(rc, rc);
1647 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1648 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1649
1650 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1651 AssertRCReturn(rc, rc);
1652 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1653 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1654
1655 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1656 {
1657 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1658 AssertRCReturn(rc, rc);
1659 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1660 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1661 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1662 }
1663
1664 return VINF_SUCCESS;
1665}
1666
1667
1668#ifdef VBOX_STRICT
1669/**
1670 * Verifies that our cached host EFER value has not changed
1671 * since we cached it.
1672 *
1673 * @param pVCpu The cross context virtual CPU structure.
1674 */
1675static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1676{
1677 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1678
1679 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1680 {
1681 uint64_t u64Val;
1682 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1683 AssertRC(rc);
1684
1685 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1686 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1687 }
1688}
1689
1690
1691/**
1692 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1693 * VMCS are correct.
1694 *
1695 * @param pVCpu The cross context virtual CPU structure.
1696 */
1697static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1698{
1699 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1700
1701 /* Verify MSR counts in the VMCS are what we think it should be. */
1702 uint32_t cMsrs;
1703 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1704 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1705
1706 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1707 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1708
1709 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1710 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1711
1712 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1713 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1714 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1715 {
1716 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1717 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1718 pGuestMsr->u32Msr, cMsrs));
1719
1720 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1721 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1722 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1723
1724 /* Verify that the permissions are as expected in the MSR bitmap. */
1725 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1726 {
1727 VMXMSREXITREAD enmRead;
1728 VMXMSREXITWRITE enmWrite;
1729 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1730 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1731 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1732 {
1733 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1734 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1735 }
1736 else
1737 {
1738 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1739 pGuestMsr->u32Msr, cMsrs));
1740 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1741 pGuestMsr->u32Msr, cMsrs));
1742 }
1743 }
1744 }
1745}
1746#endif /* VBOX_STRICT */
1747
1748
1749/**
1750 * Flushes the TLB using EPT.
1751 *
1752 * @returns VBox status code.
1753 * @param pVCpu The cross context virtual CPU structure of the calling
1754 * EMT. Can be NULL depending on @a enmFlush.
1755 * @param enmFlush Type of flush.
1756 *
1757 * @remarks Caller is responsible for making sure this function is called only
1758 * when NestedPaging is supported and providing @a enmFlush that is
1759 * supported by the CPU.
1760 * @remarks Can be called with interrupts disabled.
1761 */
1762static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1763{
1764 uint64_t au64Descriptor[2];
1765 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1766 au64Descriptor[0] = 0;
1767 else
1768 {
1769 Assert(pVCpu);
1770 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1771 }
1772 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1773
1774 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1775 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1776 rc));
1777 if ( RT_SUCCESS(rc)
1778 && pVCpu)
1779 {
1780 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1781 }
1782}
1783
1784
1785/**
1786 * Flushes the TLB using VPID.
1787 *
1788 * @returns VBox status code.
1789 * @param pVCpu The cross context virtual CPU structure of the calling
1790 * EMT. Can be NULL depending on @a enmFlush.
1791 * @param enmFlush Type of flush.
1792 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1793 * on @a enmFlush).
1794 *
1795 * @remarks Can be called with interrupts disabled.
1796 */
1797static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1798{
1799 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1800
1801 uint64_t au64Descriptor[2];
1802 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1803 {
1804 au64Descriptor[0] = 0;
1805 au64Descriptor[1] = 0;
1806 }
1807 else
1808 {
1809 AssertPtr(pVCpu);
1810 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1811 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1812 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1813 au64Descriptor[1] = GCPtr;
1814 }
1815
1816 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1817 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1818 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1819 if ( RT_SUCCESS(rc)
1820 && pVCpu)
1821 {
1822 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1823 }
1824 NOREF(rc);
1825}
1826
1827
1828/**
1829 * Invalidates a guest page by guest virtual address. Only relevant for
1830 * EPT/VPID, otherwise there is nothing really to invalidate.
1831 *
1832 * @returns VBox status code.
1833 * @param pVCpu The cross context virtual CPU structure.
1834 * @param GCVirt Guest virtual address of the page to invalidate.
1835 */
1836VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1837{
1838 AssertPtr(pVCpu);
1839 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1840
1841 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1842 if (!fFlushPending)
1843 {
1844 /*
1845 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1846 * the EPT case. See @bugref{6043} and @bugref{6177}.
1847 *
1848 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1849 * as this function maybe called in a loop with individual addresses.
1850 */
1851 PVM pVM = pVCpu->CTX_SUFF(pVM);
1852 if (pVM->hm.s.vmx.fVpid)
1853 {
1854 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1855
1856#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1857 /*
1858 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1859 * where executing INVVPID outside 64-bit mode does not flush translations of
1860 * 64-bit linear addresses, see @bugref{6208#c72}.
1861 */
1862 if (RT_HI_U32(GCVirt))
1863 fVpidFlush = false;
1864#endif
1865
1866 if (fVpidFlush)
1867 {
1868 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1869 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1870 }
1871 else
1872 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1873 }
1874 else if (pVM->hm.s.fNestedPaging)
1875 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1876 }
1877
1878 return VINF_SUCCESS;
1879}
1880
1881
1882/**
1883 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1884 * case where neither EPT nor VPID is supported by the CPU.
1885 *
1886 * @param pVCpu The cross context virtual CPU structure.
1887 * @param pCpu Pointer to the global HM struct.
1888 *
1889 * @remarks Called with interrupts disabled.
1890 */
1891static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1892{
1893 AssertPtr(pVCpu);
1894 AssertPtr(pCpu);
1895
1896 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1897
1898 Assert(pCpu->idCpu != NIL_RTCPUID);
1899 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1900 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1901 pVCpu->hm.s.fForceTLBFlush = false;
1902 return;
1903}
1904
1905
1906/**
1907 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1908 *
1909 * @param pVCpu The cross context virtual CPU structure.
1910 * @param pCpu Pointer to the global HM CPU struct.
1911 *
1912 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1913 * nomenclature. The reason is, to avoid confusion in compare statements
1914 * since the host-CPU copies are named "ASID".
1915 *
1916 * @remarks Called with interrupts disabled.
1917 */
1918static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1919{
1920#ifdef VBOX_WITH_STATISTICS
1921 bool fTlbFlushed = false;
1922# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1923# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1924 if (!fTlbFlushed) \
1925 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1926 } while (0)
1927#else
1928# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1929# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1930#endif
1931
1932 AssertPtr(pCpu);
1933 AssertPtr(pVCpu);
1934 Assert(pCpu->idCpu != NIL_RTCPUID);
1935
1936 PVM pVM = pVCpu->CTX_SUFF(pVM);
1937 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1938 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1939 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1940
1941 /*
1942 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1943 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1944 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1945 * cannot reuse the current ASID anymore.
1946 */
1947 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1948 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1949 {
1950 ++pCpu->uCurrentAsid;
1951 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1952 {
1953 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1954 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1955 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1956 }
1957
1958 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1959 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1960 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1961
1962 /*
1963 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1964 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1965 */
1966 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1967 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1968 HMVMX_SET_TAGGED_TLB_FLUSHED();
1969 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1970 }
1971
1972 /* Check for explicit TLB flushes. */
1973 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1974 {
1975 /*
1976 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1977 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1978 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1979 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1980 * mappings, see @bugref{6568}.
1981 *
1982 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1983 */
1984 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1985 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1986 HMVMX_SET_TAGGED_TLB_FLUSHED();
1987 }
1988
1989 pVCpu->hm.s.fForceTLBFlush = false;
1990 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1991
1992 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1993 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1994 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1995 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1996 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1997 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1998 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1999 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2000 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2001
2002 /* Update VMCS with the VPID. */
2003 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2004 AssertRC(rc);
2005
2006#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2007}
2008
2009
2010/**
2011 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2012 *
2013 * @returns VBox status code.
2014 * @param pVCpu The cross context virtual CPU structure.
2015 * @param pCpu Pointer to the global HM CPU struct.
2016 *
2017 * @remarks Called with interrupts disabled.
2018 */
2019static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2020{
2021 AssertPtr(pVCpu);
2022 AssertPtr(pCpu);
2023 Assert(pCpu->idCpu != NIL_RTCPUID);
2024 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2025 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2026
2027 /*
2028 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2029 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2030 */
2031 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2032 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2033 {
2034 pVCpu->hm.s.fForceTLBFlush = true;
2035 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2036 }
2037
2038 /* Check for explicit TLB flushes. */
2039 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2040 {
2041 pVCpu->hm.s.fForceTLBFlush = true;
2042 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2043 }
2044
2045 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2046 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2047
2048 if (pVCpu->hm.s.fForceTLBFlush)
2049 {
2050 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2051 pVCpu->hm.s.fForceTLBFlush = false;
2052 }
2053}
2054
2055
2056/**
2057 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2058 *
2059 * @returns VBox status code.
2060 * @param pVCpu The cross context virtual CPU structure.
2061 * @param pCpu Pointer to the global HM CPU struct.
2062 *
2063 * @remarks Called with interrupts disabled.
2064 */
2065static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2066{
2067 AssertPtr(pVCpu);
2068 AssertPtr(pCpu);
2069 Assert(pCpu->idCpu != NIL_RTCPUID);
2070 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2071 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2072
2073 /*
2074 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2075 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2076 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2077 * cannot reuse the current ASID anymore.
2078 */
2079 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2080 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2081 {
2082 pVCpu->hm.s.fForceTLBFlush = true;
2083 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2084 }
2085
2086 /* Check for explicit TLB flushes. */
2087 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2088 {
2089 /*
2090 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2091 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2092 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2093 * include fExplicitFlush's too) - an obscure corner case.
2094 */
2095 pVCpu->hm.s.fForceTLBFlush = true;
2096 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2097 }
2098
2099 PVM pVM = pVCpu->CTX_SUFF(pVM);
2100 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2101 if (pVCpu->hm.s.fForceTLBFlush)
2102 {
2103 ++pCpu->uCurrentAsid;
2104 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2105 {
2106 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2107 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2108 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2109 }
2110
2111 pVCpu->hm.s.fForceTLBFlush = false;
2112 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2113 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2114 if (pCpu->fFlushAsidBeforeUse)
2115 {
2116 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2117 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2118 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2119 {
2120 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2121 pCpu->fFlushAsidBeforeUse = false;
2122 }
2123 else
2124 {
2125 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2126 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2127 }
2128 }
2129 }
2130
2131 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2132 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2133 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2134 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2135 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2136 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2137 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2138
2139 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2140 AssertRC(rc);
2141}
2142
2143
2144/**
2145 * Flushes the guest TLB entry based on CPU capabilities.
2146 *
2147 * @param pVCpu The cross context virtual CPU structure.
2148 * @param pCpu Pointer to the global HM CPU struct.
2149 */
2150DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2151{
2152#ifdef HMVMX_ALWAYS_FLUSH_TLB
2153 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2154#endif
2155 PVM pVM = pVCpu->CTX_SUFF(pVM);
2156 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2157 {
2158 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2159 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2160 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2161 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2162 default:
2163 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2164 break;
2165 }
2166 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2167}
2168
2169
2170/**
2171 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2172 * TLB entries from the host TLB before VM-entry.
2173 *
2174 * @returns VBox status code.
2175 * @param pVM The cross context VM structure.
2176 */
2177static int hmR0VmxSetupTaggedTlb(PVM pVM)
2178{
2179 /*
2180 * Determine optimal flush type for Nested Paging.
2181 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2182 * guest execution (see hmR3InitFinalizeR0()).
2183 */
2184 if (pVM->hm.s.fNestedPaging)
2185 {
2186 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2187 {
2188 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2189 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2190 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2192 else
2193 {
2194 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2195 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2196 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2197 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2198 }
2199
2200 /* Make sure the write-back cacheable memory type for EPT is supported. */
2201 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2202 {
2203 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2204 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2205 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2206 }
2207
2208 /* EPT requires a page-walk length of 4. */
2209 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2210 {
2211 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2212 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2213 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2214 }
2215 }
2216 else
2217 {
2218 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2219 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2220 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2221 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2222 }
2223 }
2224
2225 /*
2226 * Determine optimal flush type for VPID.
2227 */
2228 if (pVM->hm.s.vmx.fVpid)
2229 {
2230 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2231 {
2232 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2233 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2234 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2235 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2236 else
2237 {
2238 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2239 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2240 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2241 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2242 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2243 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2244 pVM->hm.s.vmx.fVpid = false;
2245 }
2246 }
2247 else
2248 {
2249 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2250 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2251 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2252 pVM->hm.s.vmx.fVpid = false;
2253 }
2254 }
2255
2256 /*
2257 * Setup the handler for flushing tagged-TLBs.
2258 */
2259 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2261 else if (pVM->hm.s.fNestedPaging)
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2263 else if (pVM->hm.s.vmx.fVpid)
2264 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2265 else
2266 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2267 return VINF_SUCCESS;
2268}
2269
2270
2271/**
2272 * Sets up pin-based VM-execution controls in the VMCS.
2273 *
2274 * @returns VBox status code.
2275 * @param pVCpu The cross context virtual CPU structure.
2276 *
2277 * @remarks We don't really care about optimizing vmwrites here as it's done only
2278 * once per VM and hence we don't care about VMCS-field cache comparisons.
2279 */
2280static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2281{
2282 PVM pVM = pVCpu->CTX_SUFF(pVM);
2283 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2284 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2285
2286 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2287 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2288
2289 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2290 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2291
2292 /* Enable the VMX preemption timer. */
2293 if (pVM->hm.s.vmx.fUsePreemptTimer)
2294 {
2295 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2296 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2297 }
2298
2299#if 0
2300 /* Enable posted-interrupt processing. */
2301 if (pVM->hm.s.fPostedIntrs)
2302 {
2303 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2304 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2305 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2306 }
2307#endif
2308
2309 if ((fVal & fZap) != fVal)
2310 {
2311 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2312 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2313 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2314 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2315 }
2316
2317 /* Commit it to the VMCS and update our cache. */
2318 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2319 AssertRCReturn(rc, rc);
2320 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2321
2322 return VINF_SUCCESS;
2323}
2324
2325
2326/**
2327 * Sets up secondary processor-based VM-execution controls in the VMCS.
2328 *
2329 * @returns VBox status code.
2330 * @param pVCpu The cross context virtual CPU structure.
2331 *
2332 * @remarks We don't really care about optimizing vmwrites here as it's done only
2333 * once per VM and hence we don't care about VMCS-field cache comparisons.
2334 */
2335static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2336{
2337 PVM pVM = pVCpu->CTX_SUFF(pVM);
2338 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2339 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2340
2341 /* WBINVD causes a VM-exit. */
2342 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2343 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2344
2345 /* Enable EPT (aka nested-paging). */
2346 if (pVM->hm.s.fNestedPaging)
2347 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2348
2349 /*
2350 * Enable the INVPCID instruction if supported by the hardware and we expose
2351 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2352 */
2353 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2354 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2355 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2356
2357 /* Enable VPID. */
2358 if (pVM->hm.s.vmx.fVpid)
2359 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2360
2361 /* Enable Unrestricted guest execution. */
2362 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2363 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2364
2365#if 0
2366 if (pVM->hm.s.fVirtApicRegs)
2367 {
2368 /* Enable APIC-register virtualization. */
2369 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2371
2372 /* Enable virtual-interrupt delivery. */
2373 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2374 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2375 }
2376#endif
2377
2378 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2379 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2380 * done dynamically. */
2381 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2382 {
2383 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2384 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2385 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2386 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2387 AssertRCReturn(rc, rc);
2388 }
2389
2390 /* Enable RDTSCP. */
2391 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2392 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2393
2394 /* Enable Pause-Loop exiting. */
2395 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2396 && pVM->hm.s.vmx.cPleGapTicks
2397 && pVM->hm.s.vmx.cPleWindowTicks)
2398 {
2399 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2400
2401 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2402 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2403 AssertRCReturn(rc, rc);
2404 }
2405
2406 if ((fVal & fZap) != fVal)
2407 {
2408 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2409 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2410 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2411 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2412 }
2413
2414 /* Commit it to the VMCS and update our cache. */
2415 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2416 AssertRCReturn(rc, rc);
2417 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2418
2419 return VINF_SUCCESS;
2420}
2421
2422
2423/**
2424 * Sets up processor-based VM-execution controls in the VMCS.
2425 *
2426 * @returns VBox status code.
2427 * @param pVCpu The cross context virtual CPU structure.
2428 *
2429 * @remarks We don't really care about optimizing vmwrites here as it's done only
2430 * once per VM and hence we don't care about VMCS-field cache comparisons.
2431 */
2432static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2433{
2434 PVM pVM = pVCpu->CTX_SUFF(pVM);
2435 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2436 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2437
2438 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2441 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2442 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2443 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2444 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2445
2446 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2447 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2448 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2449 {
2450 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2451 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2452 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2453 }
2454
2455 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2456 if (!pVM->hm.s.fNestedPaging)
2457 {
2458 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2459 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2460 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2461 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2462 }
2463
2464 /* Use TPR shadowing if supported by the CPU. */
2465 if ( PDMHasApic(pVM)
2466 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2467 {
2468 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2469 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2470 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2471 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2472 AssertRCReturn(rc, rc);
2473
2474 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2475 /* CR8 writes cause a VM-exit based on TPR threshold. */
2476 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2477 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2478 }
2479 else
2480 {
2481 /*
2482 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2483 * Set this control only for 64-bit guests.
2484 */
2485 if (pVM->hm.s.fAllow64BitGuests)
2486 {
2487 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2488 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2489 }
2490 }
2491
2492 /* Use MSR-bitmaps if supported by the CPU. */
2493 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2494 {
2495 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2496
2497 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2498 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2499 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2500 AssertRCReturn(rc, rc);
2501
2502 /*
2503 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2504 * automatically using dedicated fields in the VMCS.
2505 */
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2508 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2509 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2510 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2511#if HC_ARCH_BITS == 64
2512 /*
2513 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2514 */
2515 if (pVM->hm.s.fAllow64BitGuests)
2516 {
2517 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2518 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2519 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2520 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2521 }
2522#endif
2523 /*
2524 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2525 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2526 */
2527 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2528 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2529
2530 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2531 }
2532
2533 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2534 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2535 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2536
2537 if ((fVal & fZap) != fVal)
2538 {
2539 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2540 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2541 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2542 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2543 }
2544
2545 /* Commit it to the VMCS and update our cache. */
2546 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2547 AssertRCReturn(rc, rc);
2548 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2549
2550 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2551 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2552 return hmR0VmxSetupProcCtls2(pVCpu);
2553
2554 /* Sanity check, should not really happen. */
2555 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2556 {
2557 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2558 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2559 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2560 }
2561
2562 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2563 return VINF_SUCCESS;
2564}
2565
2566
2567/**
2568 * Sets up miscellaneous (everything other than Pin & Processor-based
2569 * VM-execution) control fields in the VMCS.
2570 *
2571 * @returns VBox status code.
2572 * @param pVCpu The cross context virtual CPU structure.
2573 */
2574static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2575{
2576 AssertPtr(pVCpu);
2577
2578 int rc = VERR_GENERAL_FAILURE;
2579
2580 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2581#if 0
2582 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2583 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2584 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2585
2586 /*
2587 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2588 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2589 * We thus use the exception bitmap to control it rather than use both.
2590 */
2591 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2592 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2593
2594 /* All IO & IOIO instructions cause VM-exits. */
2595 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2596 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2597
2598 /* Initialize the MSR-bitmap area. */
2599 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2600 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2601 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2602 AssertRCReturn(rc, rc);
2603#endif
2604
2605 /* Setup MSR auto-load/store area. */
2606 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2607 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2608 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2609 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2610 AssertRCReturn(rc, rc);
2611
2612 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2613 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2615 AssertRCReturn(rc, rc);
2616
2617 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2618 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2619 AssertRCReturn(rc, rc);
2620
2621 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2622#if 0
2623 /* Setup debug controls */
2624 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2625 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2626 AssertRCReturn(rc, rc);
2627#endif
2628
2629 return rc;
2630}
2631
2632
2633/**
2634 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2635 *
2636 * We shall setup those exception intercepts that don't change during the
2637 * lifetime of the VM here. The rest are done dynamically while loading the
2638 * guest state.
2639 *
2640 * @returns VBox status code.
2641 * @param pVCpu The cross context virtual CPU structure.
2642 */
2643static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2644{
2645 AssertPtr(pVCpu);
2646
2647 uint32_t u32XcptBitmap;
2648
2649 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2650 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2651
2652 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2653 and writes, and because recursive #DBs can cause the CPU hang, we must always
2654 intercept #DB. */
2655 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2656
2657 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2658 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2659 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2660
2661 /* Commit it to the VMCS. */
2662 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2663 AssertRCReturn(rc, rc);
2664
2665 /* Update our cache of the exception bitmap. */
2666 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2667 return VINF_SUCCESS;
2668}
2669
2670
2671/**
2672 * Does per-VM VT-x initialization.
2673 *
2674 * @returns VBox status code.
2675 * @param pVM The cross context VM structure.
2676 */
2677VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2678{
2679 LogFlowFunc(("pVM=%p\n", pVM));
2680
2681 int rc = hmR0VmxStructsAlloc(pVM);
2682 if (RT_FAILURE(rc))
2683 {
2684 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2685 return rc;
2686 }
2687
2688 return VINF_SUCCESS;
2689}
2690
2691
2692/**
2693 * Does per-VM VT-x termination.
2694 *
2695 * @returns VBox status code.
2696 * @param pVM The cross context VM structure.
2697 */
2698VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2699{
2700 LogFlowFunc(("pVM=%p\n", pVM));
2701
2702#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2703 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2704 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2705#endif
2706 hmR0VmxStructsFree(pVM);
2707 return VINF_SUCCESS;
2708}
2709
2710
2711/**
2712 * Sets up the VM for execution under VT-x.
2713 * This function is only called once per-VM during initialization.
2714 *
2715 * @returns VBox status code.
2716 * @param pVM The cross context VM structure.
2717 */
2718VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2719{
2720 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2721 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2722
2723 LogFlowFunc(("pVM=%p\n", pVM));
2724
2725 /*
2726 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2727 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2728 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2729 */
2730 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2731 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2732 || !pVM->hm.s.vmx.pRealModeTSS))
2733 {
2734 LogRelFunc(("Invalid real-on-v86 state.\n"));
2735 return VERR_INTERNAL_ERROR;
2736 }
2737
2738 /* Initialize these always, see hmR3InitFinalizeR0().*/
2739 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2740 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2741
2742 /* Setup the tagged-TLB flush handlers. */
2743 int rc = hmR0VmxSetupTaggedTlb(pVM);
2744 if (RT_FAILURE(rc))
2745 {
2746 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2747 return rc;
2748 }
2749
2750 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2751 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2752#if HC_ARCH_BITS == 64
2753 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2754 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2755 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2756 {
2757 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2758 }
2759#endif
2760
2761 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2762 RTCCUINTREG uHostCR4 = ASMGetCR4();
2763 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2764 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2765
2766 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2767 {
2768 PVMCPU pVCpu = &pVM->aCpus[i];
2769 AssertPtr(pVCpu);
2770 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2771
2772 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2773 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2774
2775 /* Set revision dword at the beginning of the VMCS structure. */
2776 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2777
2778 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2779 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2780 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2781 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2782
2783 /* Load this VMCS as the current VMCS. */
2784 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2786 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2787
2788 rc = hmR0VmxSetupPinCtls(pVCpu);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2790 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupProcCtls(pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 rc = hmR0VmxSetupMiscCtls(pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800 rc = hmR0VmxInitXcptBitmap(pVCpu);
2801 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2802 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2803
2804#if HC_ARCH_BITS == 32
2805 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2806 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2807 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2808#endif
2809
2810 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2811 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2812 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2814
2815 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2816
2817 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2818 }
2819
2820 return VINF_SUCCESS;
2821}
2822
2823
2824/**
2825 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2826 * the VMCS.
2827 *
2828 * @returns VBox status code.
2829 */
2830static int hmR0VmxExportHostControlRegs(void)
2831{
2832 RTCCUINTREG uReg = ASMGetCR0();
2833 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2834 AssertRCReturn(rc, rc);
2835
2836 uReg = ASMGetCR3();
2837 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2838 AssertRCReturn(rc, rc);
2839
2840 uReg = ASMGetCR4();
2841 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2842 AssertRCReturn(rc, rc);
2843 return rc;
2844}
2845
2846
2847/**
2848 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2849 * the host-state area in the VMCS.
2850 *
2851 * @returns VBox status code.
2852 * @param pVCpu The cross context virtual CPU structure.
2853 */
2854static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2855{
2856#if HC_ARCH_BITS == 64
2857/**
2858 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2859 * requirements. See hmR0VmxExportHostSegmentRegs().
2860 */
2861# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2862 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2863 { \
2864 bool fValidSelector = true; \
2865 if ((selValue) & X86_SEL_LDT) \
2866 { \
2867 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2868 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2869 } \
2870 if (fValidSelector) \
2871 { \
2872 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2873 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2874 } \
2875 (selValue) = 0; \
2876 }
2877
2878 /*
2879 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2880 * should -not- save the messed up state without restoring the original host-state,
2881 * see @bugref{7240}.
2882 *
2883 * This apparently can happen (most likely the FPU changes), deal with it rather than
2884 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2885 */
2886 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2887 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2888 {
2889 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2890 pVCpu->idCpu));
2891 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2892 }
2893 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2894#else
2895 RT_NOREF(pVCpu);
2896#endif
2897
2898 /*
2899 * Host DS, ES, FS and GS segment registers.
2900 */
2901#if HC_ARCH_BITS == 64
2902 RTSEL uSelDS = ASMGetDS();
2903 RTSEL uSelES = ASMGetES();
2904 RTSEL uSelFS = ASMGetFS();
2905 RTSEL uSelGS = ASMGetGS();
2906#else
2907 RTSEL uSelDS = 0;
2908 RTSEL uSelES = 0;
2909 RTSEL uSelFS = 0;
2910 RTSEL uSelGS = 0;
2911#endif
2912
2913 /*
2914 * Host CS and SS segment registers.
2915 */
2916 RTSEL uSelCS = ASMGetCS();
2917 RTSEL uSelSS = ASMGetSS();
2918
2919 /*
2920 * Host TR segment register.
2921 */
2922 RTSEL uSelTR = ASMGetTR();
2923
2924#if HC_ARCH_BITS == 64
2925 /*
2926 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2927 * gain VM-entry and restore them before we get preempted.
2928 *
2929 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2930 */
2931 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2932 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2933 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2934 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2935# undef VMXLOCAL_ADJUST_HOST_SEG
2936#endif
2937
2938 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2939 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2940 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2941 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2942 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2943 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2944 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2945 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2946 Assert(uSelCS);
2947 Assert(uSelTR);
2948
2949 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2950#if 0
2951 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2952 Assert(uSelSS != 0);
2953#endif
2954
2955 /* Write these host selector fields into the host-state area in the VMCS. */
2956 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2958#if HC_ARCH_BITS == 64
2959 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2960 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2961 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2962 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2963#else
2964 NOREF(uSelDS);
2965 NOREF(uSelES);
2966 NOREF(uSelFS);
2967 NOREF(uSelGS);
2968#endif
2969 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2970 AssertRCReturn(rc, rc);
2971
2972 /*
2973 * Host GDTR and IDTR.
2974 */
2975 RTGDTR Gdtr;
2976 RTIDTR Idtr;
2977 RT_ZERO(Gdtr);
2978 RT_ZERO(Idtr);
2979 ASMGetGDTR(&Gdtr);
2980 ASMGetIDTR(&Idtr);
2981 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2982 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2983 AssertRCReturn(rc, rc);
2984
2985#if HC_ARCH_BITS == 64
2986 /*
2987 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2988 * them to the maximum limit (0xffff) on every VM-exit.
2989 */
2990 if (Gdtr.cbGdt != 0xffff)
2991 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2992
2993 /*
2994 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2995 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2996 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2997 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2998 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2999 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
3000 * at 0xffff on hosts where we are sure it won't cause trouble.
3001 */
3002# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
3003 if (Idtr.cbIdt < 0x0fff)
3004# else
3005 if (Idtr.cbIdt != 0xffff)
3006# endif
3007 {
3008 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3009 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3010 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3011 }
3012#endif
3013
3014 /*
3015 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3016 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3017 * RPL should be too in most cases.
3018 */
3019 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3020 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3021
3022 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3023#if HC_ARCH_BITS == 64
3024 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3025
3026 /*
3027 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3028 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3029 * restoration if the host has something else. Task switching is not supported in 64-bit
3030 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3031 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3032 *
3033 * [1] See Intel spec. 3.5 "System Descriptor Types".
3034 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3035 */
3036 PVM pVM = pVCpu->CTX_SUFF(pVM);
3037 Assert(pDesc->System.u4Type == 11);
3038 if ( pDesc->System.u16LimitLow != 0x67
3039 || pDesc->System.u4LimitHigh)
3040 {
3041 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3042 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3043 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3044 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3045 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3046 }
3047
3048 /*
3049 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3050 */
3051 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3052 {
3053 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3054 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3055 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3056 {
3057 /* The GDT is read-only but the writable GDT is available. */
3058 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3059 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3060 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3061 AssertRCReturn(rc, rc);
3062 }
3063 }
3064#else
3065 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3066#endif
3067 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3068 AssertRCReturn(rc, rc);
3069
3070 /*
3071 * Host FS base and GS base.
3072 */
3073#if HC_ARCH_BITS == 64
3074 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3075 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3076 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3077 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3078 AssertRCReturn(rc, rc);
3079
3080 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3081 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3082 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3083 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3084 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3085#endif
3086 return VINF_SUCCESS;
3087}
3088
3089
3090/**
3091 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3092 * host-state area of the VMCS.
3093 *
3094 * Theses MSRs will be automatically restored on the host after every successful
3095 * VM-exit.
3096 *
3097 * @returns VBox status code.
3098 * @param pVCpu The cross context virtual CPU structure.
3099 *
3100 * @remarks No-long-jump zone!!!
3101 */
3102static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3103{
3104 AssertPtr(pVCpu);
3105 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3106
3107 /*
3108 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3109 * rather than swapping them on every VM-entry.
3110 */
3111 hmR0VmxLazySaveHostMsrs(pVCpu);
3112
3113 /*
3114 * Host Sysenter MSRs.
3115 */
3116 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3117#if HC_ARCH_BITS == 32
3118 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3119 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3120#else
3121 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3122 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3123#endif
3124 AssertRCReturn(rc, rc);
3125
3126 /*
3127 * Host EFER MSR.
3128 *
3129 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3130 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3131 */
3132 PVM pVM = pVCpu->CTX_SUFF(pVM);
3133 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3134 {
3135 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3136 AssertRCReturn(rc, rc);
3137 }
3138
3139 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3140
3141 return VINF_SUCCESS;
3142}
3143
3144
3145/**
3146 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3147 *
3148 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3149 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3150 * hmR0VMxExportGuestEntryCtls().
3151 *
3152 * @returns true if we need to load guest EFER, false otherwise.
3153 * @param pVCpu The cross context virtual CPU structure.
3154 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3155 * out-of-sync. Make sure to update the required fields
3156 * before using them.
3157 *
3158 * @remarks Requires EFER, CR4.
3159 * @remarks No-long-jump zone!!!
3160 */
3161static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3162{
3163#ifdef HMVMX_ALWAYS_SWAP_EFER
3164 return true;
3165#endif
3166
3167#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3168 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3169 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3170 return false;
3171#endif
3172
3173 PVM pVM = pVCpu->CTX_SUFF(pVM);
3174 uint64_t const u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3175 uint64_t const u64GuestEfer = pMixedCtx->msrEFER;
3176
3177 /*
3178 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3179 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3180 */
3181 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3182 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3183 {
3184 return true;
3185 }
3186
3187 /*
3188 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3189 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3190 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3191 */
3192 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3193 && (pMixedCtx->cr0 & X86_CR0_PG)
3194 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3195 {
3196 /* Assert that host is PAE capable. */
3197 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3198 return true;
3199 }
3200
3201 return false;
3202}
3203
3204
3205/**
3206 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3207 *
3208 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3209 * see Intel spec. 24.8.1 "VM-entry controls".
3210 *
3211 * @returns VBox status code.
3212 * @param pVCpu The cross context virtual CPU structure.
3213 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3214 * out-of-sync. Make sure to update the required fields
3215 * before using them.
3216 *
3217 * @remarks Requires EFER.
3218 * @remarks No-long-jump zone!!!
3219 */
3220static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3221{
3222 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3223 {
3224 PVM pVM = pVCpu->CTX_SUFF(pVM);
3225 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3226 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3227
3228 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3229 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3230
3231 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3232 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3233 {
3234 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3235 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3236 }
3237 else
3238 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3239
3240 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3241 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3242 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3243 {
3244 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3245 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3246 }
3247
3248 /*
3249 * The following should -not- be set (since we're not in SMM mode):
3250 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3251 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3252 */
3253
3254 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3255 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3256
3257 if ((fVal & fZap) != fVal)
3258 {
3259 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3260 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3261 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3262 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3263 }
3264
3265 /* Commit it to the VMCS and update our cache. */
3266 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3267 {
3268 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3269 AssertRCReturn(rc, rc);
3270 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3271 }
3272
3273 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3274 }
3275 return VINF_SUCCESS;
3276}
3277
3278
3279/**
3280 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3281 *
3282 * @returns VBox status code.
3283 * @param pVCpu The cross context virtual CPU structure.
3284 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3285 * out-of-sync. Make sure to update the required fields
3286 * before using them.
3287 *
3288 * @remarks Requires EFER.
3289 */
3290static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3291{
3292 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3293 {
3294 PVM pVM = pVCpu->CTX_SUFF(pVM);
3295 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3296 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3297
3298 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3299 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3300
3301 /*
3302 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3303 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3304 * hmR0VmxExportHostMsrs().
3305 */
3306#if HC_ARCH_BITS == 64
3307 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3308 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3309#else
3310 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3311 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3312 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3313 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3314 {
3315 /* The switcher returns to long mode, EFER is managed by the switcher. */
3316 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3317 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3318 }
3319 else
3320 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3321#endif
3322
3323 /* If the newer VMCS fields for managing EFER exists, use it. */
3324 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3325 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3326 {
3327 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3328 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3329 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3330 }
3331
3332 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3333 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3334
3335 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3336 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3337 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3338
3339 /* Enable saving of the VMX preemption timer value on VM-exit. */
3340 if ( pVM->hm.s.vmx.fUsePreemptTimer
3341 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3342 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3343
3344 if ((fVal & fZap) != fVal)
3345 {
3346 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3347 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3348 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3349 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3350 }
3351
3352 /* Commit it to the VMCS and update our cache. */
3353 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3354 {
3355 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3356 AssertRCReturn(rc, rc);
3357 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3358 }
3359
3360 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3361 }
3362 return VINF_SUCCESS;
3363}
3364
3365
3366/**
3367 * Sets the TPR threshold in the VMCS.
3368 *
3369 * @returns VBox status code.
3370 * @param pVCpu The cross context virtual CPU structure.
3371 * @param u32TprThreshold The TPR threshold (task-priority class only).
3372 */
3373DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3374{
3375 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3376 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3377 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3378}
3379
3380
3381/**
3382 * Exports the guest APIC TPR state into the VMCS.
3383 *
3384 * @returns VBox status code.
3385 * @param pVCpu The cross context virtual CPU structure.
3386 *
3387 * @remarks No-long-jump zone!!!
3388 */
3389static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3390{
3391 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3392 {
3393 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3394 && APICIsEnabled(pVCpu))
3395 {
3396 /*
3397 * Setup TPR shadowing.
3398 */
3399 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3400 {
3401 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3402
3403 bool fPendingIntr = false;
3404 uint8_t u8Tpr = 0;
3405 uint8_t u8PendingIntr = 0;
3406 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3407 AssertRCReturn(rc, rc);
3408
3409 /*
3410 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3411 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3412 * priority of the pending interrupt so we can deliver the interrupt. If there
3413 * are no interrupts pending, set threshold to 0 to not cause any
3414 * TPR-below-threshold VM-exits.
3415 */
3416 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3417 uint32_t u32TprThreshold = 0;
3418 if (fPendingIntr)
3419 {
3420 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3421 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3422 const uint8_t u8TprPriority = u8Tpr >> 4;
3423 if (u8PendingPriority <= u8TprPriority)
3424 u32TprThreshold = u8PendingPriority;
3425 }
3426
3427 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3428 AssertRCReturn(rc, rc);
3429 }
3430 }
3431 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3432 }
3433 return VINF_SUCCESS;
3434}
3435
3436
3437/**
3438 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3439 *
3440 * @returns Guest's interruptibility-state.
3441 * @param pVCpu The cross context virtual CPU structure.
3442 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3443 * out-of-sync. Make sure to update the required fields
3444 * before using them.
3445 *
3446 * @remarks No-long-jump zone!!!
3447 */
3448static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3449{
3450 /*
3451 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3452 */
3453 uint32_t fIntrState = 0;
3454 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3455 {
3456 /* If inhibition is active, RIP & RFLAGS should've been accessed
3457 (i.e. read previously from the VMCS or from ring-3). */
3458#ifdef VBOX_STRICT
3459 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3460 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3461#endif
3462 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3463 {
3464 if (pMixedCtx->eflags.Bits.u1IF)
3465 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3466 else
3467 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3468 }
3469 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3470 {
3471 /*
3472 * We can clear the inhibit force flag as even if we go back to the recompiler
3473 * without executing guest code in VT-x, the flag's condition to be cleared is
3474 * met and thus the cleared state is correct.
3475 */
3476 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3477 }
3478 }
3479
3480 /*
3481 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3482 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3483 * setting this would block host-NMIs and IRET will not clear the blocking.
3484 *
3485 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3486 */
3487 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3488 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3489 {
3490 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3491 }
3492
3493 return fIntrState;
3494}
3495
3496
3497/**
3498 * Exports the guest's interruptibility-state into the guest-state area in the
3499 * VMCS.
3500 *
3501 * @returns VBox status code.
3502 * @param pVCpu The cross context virtual CPU structure.
3503 * @param fIntrState The interruptibility-state to set.
3504 */
3505static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3506{
3507 NOREF(pVCpu);
3508 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3509 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3510 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3511}
3512
3513
3514/**
3515 * Exports the exception intercepts required for guest execution in the VMCS.
3516 *
3517 * @returns VBox status code.
3518 * @param pVCpu The cross context virtual CPU structure.
3519 *
3520 * @remarks No-long-jump zone!!!
3521 */
3522static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3523{
3524 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3525 {
3526 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3527 if (pVCpu->hm.s.fGIMTrapXcptUD)
3528 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3529#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3530 else
3531 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3532#endif
3533
3534 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3535 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3536
3537 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3538 AssertRCReturn(rc, rc);
3539
3540 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3541 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3542 }
3543 return VINF_SUCCESS;
3544}
3545
3546
3547/**
3548 * Exports the guest's RIP into the guest-state area in the VMCS.
3549 *
3550 * @returns VBox status code.
3551 * @param pVCpu The cross context virtual CPU structure.
3552 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3553 * out-of-sync. Make sure to update the required fields
3554 * before using them.
3555 *
3556 * @remarks No-long-jump zone!!!
3557 */
3558static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3559{
3560 int rc = VINF_SUCCESS;
3561 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3562 {
3563 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3564 AssertRCReturn(rc, rc);
3565
3566 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3567 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3568 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3569 else
3570 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3571
3572 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3573 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3574 }
3575 return rc;
3576}
3577
3578
3579/**
3580 * Exports the guest's RSP into the guest-state area in the VMCS.
3581 *
3582 * @returns VBox status code.
3583 * @param pVCpu The cross context virtual CPU structure.
3584 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3585 * out-of-sync. Make sure to update the required fields
3586 * before using them.
3587 *
3588 * @remarks No-long-jump zone!!!
3589 */
3590static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3591{
3592 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3593 {
3594 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3595 AssertRCReturn(rc, rc);
3596
3597 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3598 }
3599 return VINF_SUCCESS;
3600}
3601
3602
3603/**
3604 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3605 *
3606 * @returns VBox status code.
3607 * @param pVCpu The cross context virtual CPU structure.
3608 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3609 * out-of-sync. Make sure to update the required fields
3610 * before using them.
3611 *
3612 * @remarks No-long-jump zone!!!
3613 */
3614static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3615{
3616 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3617 {
3618 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3619 Let us assert it as such and use 32-bit VMWRITE. */
3620 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3621 X86EFLAGS fEFlags = pMixedCtx->eflags;
3622 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3623 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3624
3625 /*
3626 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3627 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3628 * can run the real-mode guest code under Virtual 8086 mode.
3629 */
3630 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3631 {
3632 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3633 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3634 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3635 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3636 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3637 }
3638
3639 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3640 AssertRCReturn(rc, rc);
3641
3642 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3643 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3644 }
3645 return VINF_SUCCESS;
3646}
3647
3648
3649/**
3650 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3651 *
3652 * The guest FPU state is always pre-loaded hence we don't need to bother about
3653 * sharing FPU related CR0 bits between the guest and host.
3654 *
3655 * @returns VBox status code.
3656 * @param pVCpu The cross context virtual CPU structure.
3657 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3658 * out-of-sync. Make sure to update the required fields
3659 * before using them.
3660 *
3661 * @remarks No-long-jump zone!!!
3662 */
3663static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3664{
3665 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3666 {
3667 PVM pVM = pVCpu->CTX_SUFF(pVM);
3668 Assert(!RT_HI_U32(pMixedCtx->cr0));
3669 uint32_t const u32ShadowCr0 = pMixedCtx->cr0;
3670 uint32_t u32GuestCr0 = pMixedCtx->cr0;
3671
3672 /*
3673 * Setup VT-x's view of the guest CR0.
3674 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3675 */
3676 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3677 if (pVM->hm.s.fNestedPaging)
3678 {
3679 if (CPUMIsGuestPagingEnabled(pVCpu))
3680 {
3681 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3682 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3683 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3684 }
3685 else
3686 {
3687 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3688 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3689 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3690 }
3691
3692 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3693 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3694 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3695 }
3696 else
3697 {
3698 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3699 u32GuestCr0 |= X86_CR0_WP;
3700 }
3701
3702 /*
3703 * Guest FPU bits.
3704 *
3705 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3706 * using CR0.TS.
3707 *
3708 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3709 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3710 */
3711 u32GuestCr0 |= X86_CR0_NE;
3712
3713 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3714 bool const fInterceptMF = !(u32ShadowCr0 & X86_CR0_NE);
3715
3716 /*
3717 * Update exception intercepts.
3718 */
3719 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3720 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3721 {
3722 Assert(PDMVmmDevHeapIsEnabled(pVM));
3723 Assert(pVM->hm.s.vmx.pRealModeTSS);
3724 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3725 }
3726 else
3727 {
3728 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3729 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3730 if (fInterceptMF)
3731 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3732 }
3733
3734 /* Additional intercepts for debugging, define these yourself explicitly. */
3735#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3736 uXcptBitmap |= 0
3737 | RT_BIT(X86_XCPT_BP)
3738 | RT_BIT(X86_XCPT_DE)
3739 | RT_BIT(X86_XCPT_NM)
3740 | RT_BIT(X86_XCPT_TS)
3741 | RT_BIT(X86_XCPT_UD)
3742 | RT_BIT(X86_XCPT_NP)
3743 | RT_BIT(X86_XCPT_SS)
3744 | RT_BIT(X86_XCPT_GP)
3745 | RT_BIT(X86_XCPT_PF)
3746 | RT_BIT(X86_XCPT_MF)
3747 ;
3748#elif defined(HMVMX_ALWAYS_TRAP_PF)
3749 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3750#endif
3751 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3752 {
3753 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3754 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3755 }
3756 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3757
3758 /*
3759 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3760 */
3761 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3762 uint32_t fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3763 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3764 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
3765 else
3766 Assert((fSetCr0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3767
3768 u32GuestCr0 |= fSetCr0;
3769 u32GuestCr0 &= fZapCr0;
3770 u32GuestCr0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3771
3772 /*
3773 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3774 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3775 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3776 */
3777 uint32_t u32Cr0Mask = X86_CR0_PE
3778 | X86_CR0_NE
3779 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3780 | X86_CR0_PG
3781 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3782 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3783 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3784
3785 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3786 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3787 * and @bugref{6944}. */
3788#if 0
3789 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3790 u32Cr0Mask &= ~X86_CR0_PE;
3791#endif
3792 /*
3793 * Finally, update VMCS fields with the CR0 values.
3794 */
3795 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, u32GuestCr0);
3796 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, u32ShadowCr0);
3797 if (u32Cr0Mask != pVCpu->hm.s.vmx.u32Cr0Mask)
3798 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, u32Cr0Mask);
3799 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3800 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3801 AssertRCReturn(rc, rc);
3802
3803 /* Update our caches. */
3804 pVCpu->hm.s.vmx.u32Cr0Mask = u32Cr0Mask;
3805 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3806
3807 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3808
3809 Log4Func(("u32Cr0Mask=%#RX32 u32ShadowCr0=%#RX32 u32GuestCr0=%#RX32 (fSetCr0=%#RX32 fZapCr0=%#RX32\n", u32Cr0Mask,
3810 u32ShadowCr0, u32GuestCr0, fSetCr0, fZapCr0));
3811 }
3812
3813 return VINF_SUCCESS;
3814}
3815
3816
3817/**
3818 * Exports the guest control registers (CR3, CR4) into the guest-state area
3819 * in the VMCS.
3820 *
3821 * @returns VBox strict status code.
3822 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3823 * without unrestricted guest access and the VMMDev is not presently
3824 * mapped (e.g. EFI32).
3825 *
3826 * @param pVCpu The cross context virtual CPU structure.
3827 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3828 * out-of-sync. Make sure to update the required fields
3829 * before using them.
3830 *
3831 * @remarks No-long-jump zone!!!
3832 */
3833static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3834{
3835 int rc = VINF_SUCCESS;
3836 PVM pVM = pVCpu->CTX_SUFF(pVM);
3837
3838 /*
3839 * Guest CR2.
3840 * It's always loaded in the assembler code. Nothing to do here.
3841 */
3842
3843 /*
3844 * Guest CR3.
3845 */
3846 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3847 {
3848 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3849 if (pVM->hm.s.fNestedPaging)
3850 {
3851 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3852
3853 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3854 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3855 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3856 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3857
3858 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3859 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3860 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3861
3862 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3863 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3864 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3865 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3866 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3867 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3868 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3869
3870 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3871 AssertRCReturn(rc, rc);
3872
3873 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3874 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3875 {
3876 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3877 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3878 {
3879 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3880 AssertRCReturn(rc, rc);
3881 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3882 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3883 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3884 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3885 AssertRCReturn(rc, rc);
3886 }
3887
3888 /*
3889 * The guest's view of its CR3 is unblemished with Nested Paging when the
3890 * guest is using paging or we have unrestricted guest execution to handle
3891 * the guest when it's not using paging.
3892 */
3893 GCPhysGuestCR3 = pMixedCtx->cr3;
3894 }
3895 else
3896 {
3897 /*
3898 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3899 * thinks it accesses physical memory directly, we use our identity-mapped
3900 * page table to map guest-linear to guest-physical addresses. EPT takes care
3901 * of translating it to host-physical addresses.
3902 */
3903 RTGCPHYS GCPhys;
3904 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3905
3906 /* We obtain it here every time as the guest could have relocated this PCI region. */
3907 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3908 if (RT_SUCCESS(rc))
3909 { /* likely */ }
3910 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3911 {
3912 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3913 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3914 }
3915 else
3916 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3917
3918 GCPhysGuestCR3 = GCPhys;
3919 }
3920
3921 Log4Func(("u32GuestCr3=%#RGp (GstN)\n", GCPhysGuestCR3));
3922 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3923 AssertRCReturn(rc, rc);
3924 }
3925 else
3926 {
3927 /* Non-nested paging case, just use the hypervisor's CR3. */
3928 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3929
3930 Log4Func(("u32GuestCr3=%#RHv (HstN)\n", HCPhysGuestCR3));
3931 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3932 AssertRCReturn(rc, rc);
3933 }
3934
3935 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3936 }
3937
3938 /*
3939 * Guest CR4.
3940 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3941 */
3942 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3943 {
3944 Assert(!RT_HI_U32(pMixedCtx->cr4));
3945 uint32_t u32GuestCr4 = pMixedCtx->cr4;
3946 uint32_t const u32ShadowCr4 = pMixedCtx->cr4;
3947
3948 /*
3949 * Setup VT-x's view of the guest CR4.
3950 *
3951 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3952 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3953 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3954 *
3955 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3956 */
3957 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3958 {
3959 Assert(pVM->hm.s.vmx.pRealModeTSS);
3960 Assert(PDMVmmDevHeapIsEnabled(pVM));
3961 u32GuestCr4 &= ~X86_CR4_VME;
3962 }
3963
3964 if (pVM->hm.s.fNestedPaging)
3965 {
3966 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3967 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3968 {
3969 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3970 u32GuestCr4 |= X86_CR4_PSE;
3971 /* Our identity mapping is a 32-bit page directory. */
3972 u32GuestCr4 &= ~X86_CR4_PAE;
3973 }
3974 /* else use guest CR4.*/
3975 }
3976 else
3977 {
3978 /*
3979 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3980 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3981 */
3982 switch (pVCpu->hm.s.enmShadowMode)
3983 {
3984 case PGMMODE_REAL: /* Real-mode. */
3985 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3986 case PGMMODE_32_BIT: /* 32-bit paging. */
3987 {
3988 u32GuestCr4 &= ~X86_CR4_PAE;
3989 break;
3990 }
3991
3992 case PGMMODE_PAE: /* PAE paging. */
3993 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3994 {
3995 u32GuestCr4 |= X86_CR4_PAE;
3996 break;
3997 }
3998
3999 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
4000 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
4001#ifdef VBOX_ENABLE_64_BITS_GUESTS
4002 break;
4003#endif
4004 default:
4005 AssertFailed();
4006 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4007 }
4008 }
4009
4010 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4011 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4012 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4013 u32GuestCr4 |= fSetCr4;
4014 u32GuestCr4 &= fZapCr4;
4015
4016 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4017 that would cause a VM-exit. */
4018 uint32_t u32Cr4Mask = X86_CR4_VME
4019 | X86_CR4_PAE
4020 | X86_CR4_PGE
4021 | X86_CR4_PSE
4022 | X86_CR4_VMXE;
4023 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4024 u32Cr4Mask |= X86_CR4_OSXSAVE;
4025 if (pVM->cpum.ro.GuestFeatures.fPcid)
4026 u32Cr4Mask |= X86_CR4_PCIDE;
4027
4028 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4029 into the VMCS and update our cache. */
4030 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, u32GuestCr4);
4031 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, u32ShadowCr4);
4032 if (pVCpu->hm.s.vmx.u32Cr4Mask != u32Cr4Mask)
4033 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32Cr4Mask);
4034 AssertRCReturn(rc, rc);
4035 pVCpu->hm.s.vmx.u32Cr4Mask = u32Cr4Mask;
4036
4037 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4038 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4039
4040 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4041
4042 Log4Func(("u32GuestCr4=%#RX32 u32ShadowCr4=%#RX32 (fSetCr4=%#RX32 fZapCr4=%#RX32)\n", u32GuestCr4, u32ShadowCr4, fSetCr4,
4043 fZapCr4));
4044 }
4045 return rc;
4046}
4047
4048
4049/**
4050 * Exports the guest debug registers into the guest-state area in the VMCS.
4051 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4052 *
4053 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4054 *
4055 * @returns VBox status code.
4056 * @param pVCpu The cross context virtual CPU structure.
4057 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4058 * out-of-sync. Make sure to update the required fields
4059 * before using them.
4060 *
4061 * @remarks No-long-jump zone!!!
4062 */
4063static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4064{
4065 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4066
4067#ifdef VBOX_STRICT
4068 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4069 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4070 {
4071 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4072 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4073 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4074 }
4075#endif
4076
4077 bool fSteppingDB = false;
4078 bool fInterceptMovDRx = false;
4079 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4080 if (pVCpu->hm.s.fSingleInstruction)
4081 {
4082 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4083 PVM pVM = pVCpu->CTX_SUFF(pVM);
4084 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4085 {
4086 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4087 Assert(fSteppingDB == false);
4088 }
4089 else
4090 {
4091 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4092 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4093 pVCpu->hm.s.fClearTrapFlag = true;
4094 fSteppingDB = true;
4095 }
4096 }
4097
4098 uint32_t u32GuestDr7;
4099 if ( fSteppingDB
4100 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4101 {
4102 /*
4103 * Use the combined guest and host DRx values found in the hypervisor register set
4104 * because the debugger has breakpoints active or someone is single stepping on the
4105 * host side without a monitor trap flag.
4106 *
4107 * Note! DBGF expects a clean DR6 state before executing guest code.
4108 */
4109#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4110 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4111 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4112 {
4113 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4114 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4115 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4116 }
4117 else
4118#endif
4119 if (!CPUMIsHyperDebugStateActive(pVCpu))
4120 {
4121 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4122 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4123 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4124 }
4125
4126 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4127 u32GuestDr7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4128 pVCpu->hm.s.fUsingHyperDR7 = true;
4129 fInterceptMovDRx = true;
4130 }
4131 else
4132 {
4133 /*
4134 * If the guest has enabled debug registers, we need to load them prior to
4135 * executing guest code so they'll trigger at the right time.
4136 */
4137 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4138 {
4139#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4140 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4141 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4142 {
4143 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4144 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4145 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4146 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4147 }
4148 else
4149#endif
4150 if (!CPUMIsGuestDebugStateActive(pVCpu))
4151 {
4152 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4153 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4154 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4155 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4156 }
4157 Assert(!fInterceptMovDRx);
4158 }
4159 /*
4160 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4161 * must intercept #DB in order to maintain a correct DR6 guest value, and
4162 * because we need to intercept it to prevent nested #DBs from hanging the
4163 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4164 */
4165#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4166 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4167 && !CPUMIsGuestDebugStateActive(pVCpu))
4168#else
4169 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4170#endif
4171 {
4172 fInterceptMovDRx = true;
4173 }
4174
4175 /* Update DR7 with the actual guest value. */
4176 u32GuestDr7 = pMixedCtx->dr[7];
4177 pVCpu->hm.s.fUsingHyperDR7 = false;
4178 }
4179
4180 if (fInterceptMovDRx)
4181 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4182 else
4183 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4184
4185 /*
4186 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4187 * monitor-trap flag and update our cache.
4188 */
4189 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4190 {
4191 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4192 AssertRCReturn(rc2, rc2);
4193 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4194 }
4195
4196 /*
4197 * Update guest DR7.
4198 */
4199 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, u32GuestDr7);
4200 AssertRCReturn(rc, rc);
4201
4202 return VINF_SUCCESS;
4203}
4204
4205
4206#ifdef VBOX_STRICT
4207/**
4208 * Strict function to validate segment registers.
4209 *
4210 * @param pVCpu The cross context virtual CPU structure.
4211 * @param pCtx Pointer to the guest-CPU context.
4212 *
4213 * @remarks Will import guest CR0 on strict builds during validation of
4214 * segments.
4215 */
4216static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4217{
4218 /*
4219 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4220 *
4221 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4222 * because hmR0VmxExportGuestSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4223 * and doesn't change the guest-context value.
4224 */
4225 PVM pVM = pVCpu->CTX_SUFF(pVM);
4226 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4227 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4228 && ( !CPUMIsGuestInRealModeEx(pCtx)
4229 && !CPUMIsGuestInV86ModeEx(pCtx)))
4230 {
4231 /* Protected mode checks */
4232 /* CS */
4233 Assert(pCtx->cs.Attr.n.u1Present);
4234 Assert(!(pCtx->cs.Attr.u & 0xf00));
4235 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4236 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4237 || !(pCtx->cs.Attr.n.u1Granularity));
4238 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4239 || (pCtx->cs.Attr.n.u1Granularity));
4240 /* CS cannot be loaded with NULL in protected mode. */
4241 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4242 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4243 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4244 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4245 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4246 else
4247 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4248 /* SS */
4249 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4250 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4251 if ( !(pCtx->cr0 & X86_CR0_PE)
4252 || pCtx->cs.Attr.n.u4Type == 3)
4253 {
4254 Assert(!pCtx->ss.Attr.n.u2Dpl);
4255 }
4256 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4257 {
4258 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4259 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4260 Assert(pCtx->ss.Attr.n.u1Present);
4261 Assert(!(pCtx->ss.Attr.u & 0xf00));
4262 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4263 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4264 || !(pCtx->ss.Attr.n.u1Granularity));
4265 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4266 || (pCtx->ss.Attr.n.u1Granularity));
4267 }
4268 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmentReg(). */
4269 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4270 {
4271 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4272 Assert(pCtx->ds.Attr.n.u1Present);
4273 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4274 Assert(!(pCtx->ds.Attr.u & 0xf00));
4275 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4276 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4277 || !(pCtx->ds.Attr.n.u1Granularity));
4278 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4279 || (pCtx->ds.Attr.n.u1Granularity));
4280 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4281 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4282 }
4283 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4284 {
4285 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4286 Assert(pCtx->es.Attr.n.u1Present);
4287 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4288 Assert(!(pCtx->es.Attr.u & 0xf00));
4289 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4290 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4291 || !(pCtx->es.Attr.n.u1Granularity));
4292 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4293 || (pCtx->es.Attr.n.u1Granularity));
4294 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4295 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4296 }
4297 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4298 {
4299 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4300 Assert(pCtx->fs.Attr.n.u1Present);
4301 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4302 Assert(!(pCtx->fs.Attr.u & 0xf00));
4303 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4304 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4305 || !(pCtx->fs.Attr.n.u1Granularity));
4306 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4307 || (pCtx->fs.Attr.n.u1Granularity));
4308 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4309 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4310 }
4311 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4312 {
4313 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4314 Assert(pCtx->gs.Attr.n.u1Present);
4315 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4316 Assert(!(pCtx->gs.Attr.u & 0xf00));
4317 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4318 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4319 || !(pCtx->gs.Attr.n.u1Granularity));
4320 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4321 || (pCtx->gs.Attr.n.u1Granularity));
4322 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4323 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4324 }
4325 /* 64-bit capable CPUs. */
4326# if HC_ARCH_BITS == 64
4327 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4328 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4329 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4330 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4331# endif
4332 }
4333 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4334 || ( CPUMIsGuestInRealModeEx(pCtx)
4335 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4336 {
4337 /* Real and v86 mode checks. */
4338 /* hmR0VmxExportGuestSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4339 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4340 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4341 {
4342 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4343 }
4344 else
4345 {
4346 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4347 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4348 }
4349
4350 /* CS */
4351 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4352 Assert(pCtx->cs.u32Limit == 0xffff);
4353 Assert(u32CSAttr == 0xf3);
4354 /* SS */
4355 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4356 Assert(pCtx->ss.u32Limit == 0xffff);
4357 Assert(u32SSAttr == 0xf3);
4358 /* DS */
4359 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4360 Assert(pCtx->ds.u32Limit == 0xffff);
4361 Assert(u32DSAttr == 0xf3);
4362 /* ES */
4363 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4364 Assert(pCtx->es.u32Limit == 0xffff);
4365 Assert(u32ESAttr == 0xf3);
4366 /* FS */
4367 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4368 Assert(pCtx->fs.u32Limit == 0xffff);
4369 Assert(u32FSAttr == 0xf3);
4370 /* GS */
4371 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4372 Assert(pCtx->gs.u32Limit == 0xffff);
4373 Assert(u32GSAttr == 0xf3);
4374 /* 64-bit capable CPUs. */
4375# if HC_ARCH_BITS == 64
4376 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4377 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4378 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4379 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4380# endif
4381 }
4382}
4383#endif /* VBOX_STRICT */
4384
4385
4386/**
4387 * Exports a guest segment register into the guest-state area in the VMCS.
4388 *
4389 * @returns VBox status code.
4390 * @param pVCpu The cross context virtual CPU structure.
4391 * @param idxSel Index of the selector in the VMCS.
4392 * @param idxLimit Index of the segment limit in the VMCS.
4393 * @param idxBase Index of the segment base in the VMCS.
4394 * @param idxAccess Index of the access rights of the segment in the VMCS.
4395 * @param pSelReg Pointer to the segment selector.
4396 *
4397 * @remarks No-long-jump zone!!!
4398 */
4399static int hmR0VmxExportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
4400 PCCPUMSELREG pSelReg)
4401{
4402 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4403 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4404 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4405 AssertRCReturn(rc, rc);
4406
4407 uint32_t u32Access = pSelReg->Attr.u;
4408 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4409 {
4410 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4411 u32Access = 0xf3;
4412 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4413 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4414 }
4415 else
4416 {
4417 /*
4418 * The way to differentiate between whether this is really a null selector or was just
4419 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4420 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4421 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4422 * NULL selectors loaded in protected-mode have their attribute as 0.
4423 */
4424 if (!u32Access)
4425 u32Access = X86DESCATTR_UNUSABLE;
4426 }
4427
4428 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4429 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4430 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4431
4432 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4433 AssertRCReturn(rc, rc);
4434 return rc;
4435}
4436
4437
4438/**
4439 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4440 * into the guest-state area in the VMCS.
4441 *
4442 * @returns VBox status code.
4443 * @param pVCpu The cross context virtual CPU structure.
4444 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4445 * out-of-sync. Make sure to update the required fields
4446 * before using them.
4447 *
4448 * @remarks Will import guest CR0 on strict builds during validation of
4449 * segments.
4450 * @remarks No-long-jump zone!!!
4451 */
4452static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4453{
4454 int rc = VERR_INTERNAL_ERROR_5;
4455 PVM pVM = pVCpu->CTX_SUFF(pVM);
4456
4457 /*
4458 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4459 */
4460 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4461 {
4462#ifdef VBOX_WITH_REM
4463 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4464 {
4465 Assert(pVM->hm.s.vmx.pRealModeTSS);
4466 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4467 if ( pVCpu->hm.s.vmx.fWasInRealMode
4468 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4469 {
4470 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4471 in real-mode (e.g. OpenBSD 4.0) */
4472 REMFlushTBs(pVM);
4473 Log4Func(("Switch to protected mode detected!\n"));
4474 pVCpu->hm.s.vmx.fWasInRealMode = false;
4475 }
4476 }
4477#endif
4478 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
4479 {
4480 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4481 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4482 rc = HMVMX_EXPORT_SREG(CS, &pMixedCtx->cs);
4483 AssertRCReturn(rc, rc);
4484 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CS);
4485 }
4486
4487 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SS)
4488 {
4489 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4490 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4491 rc = HMVMX_EXPORT_SREG(SS, &pMixedCtx->ss);
4492 AssertRCReturn(rc, rc);
4493 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SS);
4494 }
4495
4496 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_DS)
4497 {
4498 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4499 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4500 rc = HMVMX_EXPORT_SREG(DS, &pMixedCtx->ds);
4501 AssertRCReturn(rc, rc);
4502 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_DS);
4503 }
4504
4505 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_ES)
4506 {
4507 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4508 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4509 rc = HMVMX_EXPORT_SREG(ES, &pMixedCtx->es);
4510 AssertRCReturn(rc, rc);
4511 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_ES);
4512 }
4513
4514 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_FS)
4515 {
4516 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4517 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4518 rc = HMVMX_EXPORT_SREG(FS, &pMixedCtx->fs);
4519 AssertRCReturn(rc, rc);
4520 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_FS);
4521 }
4522
4523 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GS)
4524 {
4525 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4526 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4527 rc = HMVMX_EXPORT_SREG(GS, &pMixedCtx->gs);
4528 AssertRCReturn(rc, rc);
4529 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GS);
4530 }
4531
4532#ifdef VBOX_STRICT
4533 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4534#endif
4535
4536 /* Update the exit history entry with the correct CS.BASE + RIP. */
4537 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4538 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4539
4540 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4541 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4542 }
4543
4544 /*
4545 * Guest TR.
4546 */
4547 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4548 {
4549 /*
4550 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4551 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4552 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4553 */
4554 uint16_t u16Sel = 0;
4555 uint32_t u32Limit = 0;
4556 uint64_t u64Base = 0;
4557 uint32_t u32AccessRights = 0;
4558
4559 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4560 {
4561 u16Sel = pMixedCtx->tr.Sel;
4562 u32Limit = pMixedCtx->tr.u32Limit;
4563 u64Base = pMixedCtx->tr.u64Base;
4564 u32AccessRights = pMixedCtx->tr.Attr.u;
4565 }
4566 else
4567 {
4568 Assert(pVM->hm.s.vmx.pRealModeTSS);
4569 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4570
4571 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4572 RTGCPHYS GCPhys;
4573 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4574 AssertRCReturn(rc, rc);
4575
4576 X86DESCATTR DescAttr;
4577 DescAttr.u = 0;
4578 DescAttr.n.u1Present = 1;
4579 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4580
4581 u16Sel = 0;
4582 u32Limit = HM_VTX_TSS_SIZE;
4583 u64Base = GCPhys; /* in real-mode phys = virt. */
4584 u32AccessRights = DescAttr.u;
4585 }
4586
4587 /* Validate. */
4588 Assert(!(u16Sel & RT_BIT(2)));
4589 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4590 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4591 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4592 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4593 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4594 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4595 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4596 Assert( (u32Limit & 0xfff) == 0xfff
4597 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4598 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4599 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4600
4601 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4602 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4603 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4604 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4605 AssertRCReturn(rc, rc);
4606
4607 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4608 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4609 }
4610
4611 /*
4612 * Guest GDTR.
4613 */
4614 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4615 {
4616 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4617 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4618 AssertRCReturn(rc, rc);
4619
4620 /* Validate. */
4621 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4622
4623 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4624 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4625 }
4626
4627 /*
4628 * Guest LDTR.
4629 */
4630 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4631 {
4632 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4633 uint32_t u32Access = 0;
4634 if (!pMixedCtx->ldtr.Attr.u)
4635 u32Access = X86DESCATTR_UNUSABLE;
4636 else
4637 u32Access = pMixedCtx->ldtr.Attr.u;
4638
4639 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4640 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4641 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4642 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4643 AssertRCReturn(rc, rc);
4644
4645 /* Validate. */
4646 if (!(u32Access & X86DESCATTR_UNUSABLE))
4647 {
4648 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4649 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4650 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4651 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4652 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4653 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4654 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4655 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4656 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4657 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4658 }
4659
4660 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4661 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4662 }
4663
4664 /*
4665 * Guest IDTR.
4666 */
4667 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4668 {
4669 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4670 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4671 AssertRCReturn(rc, rc);
4672
4673 /* Validate. */
4674 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4675
4676 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4677 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4678 }
4679
4680 return VINF_SUCCESS;
4681}
4682
4683
4684/**
4685 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4686 * areas.
4687 *
4688 * These MSRs will automatically be loaded to the host CPU on every successful
4689 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4690 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4691 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4692 *
4693 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4694 *
4695 * @returns VBox status code.
4696 * @param pVCpu The cross context virtual CPU structure.
4697 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4698 * out-of-sync. Make sure to update the required fields
4699 * before using them.
4700 *
4701 * @remarks No-long-jump zone!!!
4702 */
4703static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4704{
4705 AssertPtr(pVCpu);
4706 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4707
4708 /*
4709 * MSRs that we use the auto-load/store MSR area in the VMCS.
4710 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4711 */
4712 PVM pVM = pVCpu->CTX_SUFF(pVM);
4713 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4714 {
4715 if (pVM->hm.s.fAllow64BitGuests)
4716 {
4717#if HC_ARCH_BITS == 32
4718 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4719 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4720 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4721 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4722 AssertRCReturn(rc, rc);
4723# ifdef LOG_ENABLED
4724 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4725 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4726 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4727# endif
4728#endif
4729 }
4730 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4731 }
4732
4733 /*
4734 * Guest Sysenter MSRs.
4735 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4736 * VM-exits on WRMSRs for these MSRs.
4737 */
4738 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4739 {
4740 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4741 {
4742 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4743 AssertRCReturn(rc, rc);
4744 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4745 }
4746
4747 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4748 {
4749 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4750 AssertRCReturn(rc, rc);
4751 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4752 }
4753
4754 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4755 {
4756 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4757 AssertRCReturn(rc, rc);
4758 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4759 }
4760 }
4761
4762 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4763 {
4764 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4765 {
4766 /*
4767 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4768 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4769 */
4770 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4771 {
4772 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4773 AssertRCReturn(rc,rc);
4774 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4775 }
4776 else
4777 {
4778 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4779 NULL /* pfAddedAndUpdated */);
4780 AssertRCReturn(rc, rc);
4781
4782 /* We need to intercept reads too, see @bugref{7386#c16}. */
4783 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4784 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4785 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4786 pVCpu->hm.s.vmx.cMsrs));
4787 }
4788 }
4789 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4790 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4791 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4792 }
4793
4794 return VINF_SUCCESS;
4795}
4796
4797
4798#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4799/**
4800 * Check if guest state allows safe use of 32-bit switcher again.
4801 *
4802 * Segment bases and protected mode structures must be 32-bit addressable
4803 * because the 32-bit switcher will ignore high dword when writing these VMCS
4804 * fields. See @bugref{8432} for details.
4805 *
4806 * @returns true if safe, false if must continue to use the 64-bit switcher.
4807 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4808 * out-of-sync. Make sure to update the required fields
4809 * before using them.
4810 *
4811 * @remarks No-long-jump zone!!!
4812 */
4813static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4814{
4815 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4816 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4817 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4818 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4819 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4820 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4821 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4822 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4823 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4824 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4825
4826 /* All good, bases are 32-bit. */
4827 return true;
4828}
4829#endif
4830
4831
4832/**
4833 * Selects up the appropriate function to run guest code.
4834 *
4835 * @returns VBox status code.
4836 * @param pVCpu The cross context virtual CPU structure.
4837 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4838 * out-of-sync. Make sure to update the required fields
4839 * before using them.
4840 *
4841 * @remarks No-long-jump zone!!!
4842 */
4843static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4844{
4845 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4846 {
4847#ifndef VBOX_ENABLE_64_BITS_GUESTS
4848 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4849#endif
4850 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4851#if HC_ARCH_BITS == 32
4852 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4853 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4854 {
4855#ifdef VBOX_STRICT
4856 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4857 {
4858 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4859 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4860 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4861 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4862 | HM_CHANGED_VMX_ENTRY_CTLS
4863 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4864 }
4865#endif
4866 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4867
4868 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4869 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4870 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4871 Log4Func(("Selected 64-bit switcher\n"));
4872 }
4873#else
4874 /* 64-bit host. */
4875 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4876#endif
4877 }
4878 else
4879 {
4880 /* Guest is not in long mode, use the 32-bit handler. */
4881#if HC_ARCH_BITS == 32
4882 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4883 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4884 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4885 {
4886# ifdef VBOX_STRICT
4887 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4888 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4889 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
4890 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4891 | HM_CHANGED_VMX_ENTRY_CTLS
4892 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4893# endif
4894 }
4895# ifdef VBOX_ENABLE_64_BITS_GUESTS
4896 /*
4897 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4898 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4899 * switcher flag because now we know the guest is in a sane state where it's safe
4900 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4901 * the much faster 32-bit switcher again.
4902 */
4903 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4904 {
4905 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4906 Log4Func(("Selected 32-bit switcher\n"));
4907 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4908 }
4909 else
4910 {
4911 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4912 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4913 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4914 {
4915 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4916 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4917 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4918 | HM_CHANGED_VMX_ENTRY_CTLS
4919 | HM_CHANGED_VMX_EXIT_CTLS
4920 | HM_CHANGED_HOST_CONTEXT);
4921 Log4Func(("Selected 32-bit switcher (safe)\n"));
4922 }
4923 }
4924# else
4925 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4926# endif
4927#else
4928 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4929#endif
4930 }
4931 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4932 return VINF_SUCCESS;
4933}
4934
4935
4936/**
4937 * Wrapper for running the guest code in VT-x.
4938 *
4939 * @returns VBox status code, no informational status codes.
4940 * @param pVCpu The cross context virtual CPU structure.
4941 * @param pCtx Pointer to the guest-CPU context.
4942 *
4943 * @remarks No-long-jump zone!!!
4944 */
4945DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4946{
4947 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4948 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4949
4950 /*
4951 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4952 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4953 * callee-saved and thus the need for this XMM wrapper.
4954 *
4955 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4956 */
4957 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4958 /** @todo Add stats for resume vs launch. */
4959 PVM pVM = pVCpu->CTX_SUFF(pVM);
4960#ifdef VBOX_WITH_KERNEL_USING_XMM
4961 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4962#else
4963 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4964#endif
4965 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4966 return rc;
4967}
4968
4969
4970/**
4971 * Reports world-switch error and dumps some useful debug info.
4972 *
4973 * @param pVCpu The cross context virtual CPU structure.
4974 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4975 * @param pCtx Pointer to the guest-CPU context.
4976 * @param pVmxTransient Pointer to the VMX transient structure (only
4977 * exitReason updated).
4978 */
4979static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4980{
4981 Assert(pVCpu);
4982 Assert(pCtx);
4983 Assert(pVmxTransient);
4984 HMVMX_ASSERT_PREEMPT_SAFE();
4985
4986 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4987 switch (rcVMRun)
4988 {
4989 case VERR_VMX_INVALID_VMXON_PTR:
4990 AssertFailed();
4991 break;
4992 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4993 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4994 {
4995 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4996 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4997 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4998 AssertRC(rc);
4999
5000 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
5001 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
5002 Cannot do it here as we may have been long preempted. */
5003
5004#ifdef VBOX_STRICT
5005 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
5006 pVmxTransient->uExitReason));
5007 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
5008 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
5009 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
5010 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
5011 else
5012 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
5013 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
5014 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
5015
5016 /* VMX control bits. */
5017 uint32_t u32Val;
5018 uint64_t u64Val;
5019 RTHCUINTREG uHCReg;
5020 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
5021 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
5022 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
5023 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
5024 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
5025 {
5026 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
5027 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
5028 }
5029 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
5030 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
5031 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
5032 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
5033 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5034 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5035 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5036 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5037 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5038 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5039 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5040 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5041 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5042 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5043 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5044 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5045 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5046 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5047 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5048 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5049 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5050 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5051 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5052 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5053 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5054 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5055 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5056 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5057 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5058 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5059 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5060 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5061 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5062 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5063 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5064 {
5065 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5066 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5067 }
5068
5069 /* Guest bits. */
5070 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5071 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5072 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5073 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5074 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5075 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5076 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5077 {
5078 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5079 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5080 }
5081
5082 /* Host bits. */
5083 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5084 Log4(("Host CR0 %#RHr\n", uHCReg));
5085 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5086 Log4(("Host CR3 %#RHr\n", uHCReg));
5087 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5088 Log4(("Host CR4 %#RHr\n", uHCReg));
5089
5090 RTGDTR HostGdtr;
5091 PCX86DESCHC pDesc;
5092 ASMGetGDTR(&HostGdtr);
5093 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5094 Log4(("Host CS %#08x\n", u32Val));
5095 if (u32Val < HostGdtr.cbGdt)
5096 {
5097 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5098 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5099 }
5100
5101 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5102 Log4(("Host DS %#08x\n", u32Val));
5103 if (u32Val < HostGdtr.cbGdt)
5104 {
5105 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5106 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5107 }
5108
5109 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5110 Log4(("Host ES %#08x\n", u32Val));
5111 if (u32Val < HostGdtr.cbGdt)
5112 {
5113 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5114 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5115 }
5116
5117 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5118 Log4(("Host FS %#08x\n", u32Val));
5119 if (u32Val < HostGdtr.cbGdt)
5120 {
5121 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5122 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5123 }
5124
5125 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5126 Log4(("Host GS %#08x\n", u32Val));
5127 if (u32Val < HostGdtr.cbGdt)
5128 {
5129 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5130 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5131 }
5132
5133 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5134 Log4(("Host SS %#08x\n", u32Val));
5135 if (u32Val < HostGdtr.cbGdt)
5136 {
5137 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5138 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5139 }
5140
5141 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5142 Log4(("Host TR %#08x\n", u32Val));
5143 if (u32Val < HostGdtr.cbGdt)
5144 {
5145 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5146 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5147 }
5148
5149 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5150 Log4(("Host TR Base %#RHv\n", uHCReg));
5151 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5152 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5153 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5154 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5155 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5156 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5157 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5158 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5159 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5160 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5161 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5162 Log4(("Host RSP %#RHv\n", uHCReg));
5163 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5164 Log4(("Host RIP %#RHv\n", uHCReg));
5165# if HC_ARCH_BITS == 64
5166 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5167 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5168 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5169 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5170 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5171 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5172# endif
5173#endif /* VBOX_STRICT */
5174 break;
5175 }
5176
5177 default:
5178 /* Impossible */
5179 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5180 break;
5181 }
5182 NOREF(pCtx);
5183}
5184
5185
5186#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5187#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5188# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5189#endif
5190#ifdef VBOX_STRICT
5191static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5192{
5193 switch (idxField)
5194 {
5195 case VMX_VMCS_GUEST_RIP:
5196 case VMX_VMCS_GUEST_RSP:
5197 case VMX_VMCS_GUEST_SYSENTER_EIP:
5198 case VMX_VMCS_GUEST_SYSENTER_ESP:
5199 case VMX_VMCS_GUEST_GDTR_BASE:
5200 case VMX_VMCS_GUEST_IDTR_BASE:
5201 case VMX_VMCS_GUEST_CS_BASE:
5202 case VMX_VMCS_GUEST_DS_BASE:
5203 case VMX_VMCS_GUEST_ES_BASE:
5204 case VMX_VMCS_GUEST_FS_BASE:
5205 case VMX_VMCS_GUEST_GS_BASE:
5206 case VMX_VMCS_GUEST_SS_BASE:
5207 case VMX_VMCS_GUEST_LDTR_BASE:
5208 case VMX_VMCS_GUEST_TR_BASE:
5209 case VMX_VMCS_GUEST_CR3:
5210 return true;
5211 }
5212 return false;
5213}
5214
5215static bool hmR0VmxIsValidReadField(uint32_t idxField)
5216{
5217 switch (idxField)
5218 {
5219 /* Read-only fields. */
5220 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5221 return true;
5222 }
5223 /* Remaining readable fields should also be writable. */
5224 return hmR0VmxIsValidWriteField(idxField);
5225}
5226#endif /* VBOX_STRICT */
5227
5228
5229/**
5230 * Executes the specified handler in 64-bit mode.
5231 *
5232 * @returns VBox status code (no informational status codes).
5233 * @param pVCpu The cross context virtual CPU structure.
5234 * @param enmOp The operation to perform.
5235 * @param cParams Number of parameters.
5236 * @param paParam Array of 32-bit parameters.
5237 */
5238VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5239{
5240 PVM pVM = pVCpu->CTX_SUFF(pVM);
5241 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5242 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5243 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5244 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5245
5246#ifdef VBOX_STRICT
5247 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5248 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5249
5250 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5251 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5252#endif
5253
5254 /* Disable interrupts. */
5255 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5256
5257#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5258 RTCPUID idHostCpu = RTMpCpuId();
5259 CPUMR0SetLApic(pVCpu, idHostCpu);
5260#endif
5261
5262 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5263 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5264
5265 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5266 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5267 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5268
5269 /* Leave VMX Root Mode. */
5270 VMXDisable();
5271
5272 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5273
5274 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5275 CPUMSetHyperEIP(pVCpu, enmOp);
5276 for (int i = (int)cParams - 1; i >= 0; i--)
5277 CPUMPushHyper(pVCpu, paParam[i]);
5278
5279 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5280
5281 /* Call the switcher. */
5282 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5283 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5284
5285 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5286 /* Make sure the VMX instructions don't cause #UD faults. */
5287 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5288
5289 /* Re-enter VMX Root Mode */
5290 int rc2 = VMXEnable(HCPhysCpuPage);
5291 if (RT_FAILURE(rc2))
5292 {
5293 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5294 ASMSetFlags(fOldEFlags);
5295 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5296 return rc2;
5297 }
5298
5299 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5300 AssertRC(rc2);
5301 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5302 Assert(!(ASMGetFlags() & X86_EFL_IF));
5303 ASMSetFlags(fOldEFlags);
5304 return rc;
5305}
5306
5307
5308/**
5309 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5310 * supporting 64-bit guests.
5311 *
5312 * @returns VBox status code.
5313 * @param fResume Whether to VMLAUNCH or VMRESUME.
5314 * @param pCtx Pointer to the guest-CPU context.
5315 * @param pCache Pointer to the VMCS cache.
5316 * @param pVM The cross context VM structure.
5317 * @param pVCpu The cross context virtual CPU structure.
5318 */
5319DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5320{
5321 NOREF(fResume);
5322
5323 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5324 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5325
5326#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5327 pCache->uPos = 1;
5328 pCache->interPD = PGMGetInterPaeCR3(pVM);
5329 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5330#endif
5331
5332#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5333 pCache->TestIn.HCPhysCpuPage = 0;
5334 pCache->TestIn.HCPhysVmcs = 0;
5335 pCache->TestIn.pCache = 0;
5336 pCache->TestOut.HCPhysVmcs = 0;
5337 pCache->TestOut.pCache = 0;
5338 pCache->TestOut.pCtx = 0;
5339 pCache->TestOut.eflags = 0;
5340#else
5341 NOREF(pCache);
5342#endif
5343
5344 uint32_t aParam[10];
5345 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5346 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5347 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5348 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5349 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5350 aParam[5] = 0;
5351 aParam[6] = VM_RC_ADDR(pVM, pVM);
5352 aParam[7] = 0;
5353 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5354 aParam[9] = 0;
5355
5356#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5357 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5358 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5359#endif
5360 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5361
5362#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5363 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5364 Assert(pCtx->dr[4] == 10);
5365 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5366#endif
5367
5368#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5369 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5370 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5371 pVCpu->hm.s.vmx.HCPhysVmcs));
5372 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5373 pCache->TestOut.HCPhysVmcs));
5374 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5375 pCache->TestOut.pCache));
5376 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5377 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5378 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5379 pCache->TestOut.pCtx));
5380 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5381#endif
5382 NOREF(pCtx);
5383 return rc;
5384}
5385
5386
5387/**
5388 * Initialize the VMCS-Read cache.
5389 *
5390 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5391 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5392 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5393 * (those that have a 32-bit FULL & HIGH part).
5394 *
5395 * @returns VBox status code.
5396 * @param pVCpu The cross context virtual CPU structure.
5397 */
5398static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5399{
5400#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5401 do { \
5402 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5403 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5404 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5405 ++cReadFields; \
5406 } while (0)
5407
5408 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5409 uint32_t cReadFields = 0;
5410
5411 /*
5412 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5413 * and serve to indicate exceptions to the rules.
5414 */
5415
5416 /* Guest-natural selector base fields. */
5417#if 0
5418 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5420 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5421#endif
5422 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5424 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5425 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5427 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5428 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5429 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5430 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5432 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5433 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5434#if 0
5435 /* Unused natural width guest-state fields. */
5436 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5437 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5438#endif
5439 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5440 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5441
5442 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5443 these 64-bit fields (using "FULL" and "HIGH" fields). */
5444#if 0
5445 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5446 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5447 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5448 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5449 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5450 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5451 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5452 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5453 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5454#endif
5455
5456 /* Natural width guest-state fields. */
5457 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5458#if 0
5459 /* Currently unused field. */
5460 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5461#endif
5462
5463 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5464 {
5465 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5466 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5467 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5468 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5469 }
5470 else
5471 {
5472 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5473 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5474 }
5475
5476#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5477 return VINF_SUCCESS;
5478}
5479
5480
5481/**
5482 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5483 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5484 * darwin, running 64-bit guests).
5485 *
5486 * @returns VBox status code.
5487 * @param pVCpu The cross context virtual CPU structure.
5488 * @param idxField The VMCS field encoding.
5489 * @param u64Val 16, 32 or 64-bit value.
5490 */
5491VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5492{
5493 int rc;
5494 switch (idxField)
5495 {
5496 /*
5497 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5498 */
5499 /* 64-bit Control fields. */
5500 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5501 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5502 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5503 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5504 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5505 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5506 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5507 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5508 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5509 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5510 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5511 case VMX_VMCS64_CTRL_EPTP_FULL:
5512 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5513 /* 64-bit Guest-state fields. */
5514 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5515 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5516 case VMX_VMCS64_GUEST_PAT_FULL:
5517 case VMX_VMCS64_GUEST_EFER_FULL:
5518 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5519 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5520 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5521 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5522 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5523 /* 64-bit Host-state fields. */
5524 case VMX_VMCS64_HOST_PAT_FULL:
5525 case VMX_VMCS64_HOST_EFER_FULL:
5526 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5527 {
5528 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5529 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5530 break;
5531 }
5532
5533 /*
5534 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5535 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5536 */
5537 /* Natural-width Guest-state fields. */
5538 case VMX_VMCS_GUEST_CR3:
5539 case VMX_VMCS_GUEST_ES_BASE:
5540 case VMX_VMCS_GUEST_CS_BASE:
5541 case VMX_VMCS_GUEST_SS_BASE:
5542 case VMX_VMCS_GUEST_DS_BASE:
5543 case VMX_VMCS_GUEST_FS_BASE:
5544 case VMX_VMCS_GUEST_GS_BASE:
5545 case VMX_VMCS_GUEST_LDTR_BASE:
5546 case VMX_VMCS_GUEST_TR_BASE:
5547 case VMX_VMCS_GUEST_GDTR_BASE:
5548 case VMX_VMCS_GUEST_IDTR_BASE:
5549 case VMX_VMCS_GUEST_RSP:
5550 case VMX_VMCS_GUEST_RIP:
5551 case VMX_VMCS_GUEST_SYSENTER_ESP:
5552 case VMX_VMCS_GUEST_SYSENTER_EIP:
5553 {
5554 if (!(RT_HI_U32(u64Val)))
5555 {
5556 /* If this field is 64-bit, VT-x will zero out the top bits. */
5557 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5558 }
5559 else
5560 {
5561 /* Assert that only the 32->64 switcher case should ever come here. */
5562 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5563 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5564 }
5565 break;
5566 }
5567
5568 default:
5569 {
5570 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5571 rc = VERR_INVALID_PARAMETER;
5572 break;
5573 }
5574 }
5575 AssertRCReturn(rc, rc);
5576 return rc;
5577}
5578
5579
5580/**
5581 * Queue up a VMWRITE by using the VMCS write cache.
5582 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5583 *
5584 * @param pVCpu The cross context virtual CPU structure.
5585 * @param idxField The VMCS field encoding.
5586 * @param u64Val 16, 32 or 64-bit value.
5587 */
5588VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5589{
5590 AssertPtr(pVCpu);
5591 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5592
5593 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5594 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5595
5596 /* Make sure there are no duplicates. */
5597 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5598 {
5599 if (pCache->Write.aField[i] == idxField)
5600 {
5601 pCache->Write.aFieldVal[i] = u64Val;
5602 return VINF_SUCCESS;
5603 }
5604 }
5605
5606 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5607 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5608 pCache->Write.cValidEntries++;
5609 return VINF_SUCCESS;
5610}
5611#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5612
5613
5614/**
5615 * Sets up the usage of TSC-offsetting and updates the VMCS.
5616 *
5617 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5618 * VMX preemption timer.
5619 *
5620 * @returns VBox status code.
5621 * @param pVCpu The cross context virtual CPU structure.
5622 *
5623 * @remarks No-long-jump zone!!!
5624 */
5625static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5626{
5627 bool fOffsettedTsc;
5628 bool fParavirtTsc;
5629 PVM pVM = pVCpu->CTX_SUFF(pVM);
5630 uint64_t uTscOffset;
5631 if (pVM->hm.s.vmx.fUsePreemptTimer)
5632 {
5633 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5634
5635 /* Make sure the returned values have sane upper and lower boundaries. */
5636 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5637 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5638 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5639 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5640
5641 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5642 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5643 AssertRC(rc);
5644 }
5645 else
5646 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5647
5648 /** @todo later optimize this to be done elsewhere and not before every
5649 * VM-entry. */
5650 if (fParavirtTsc)
5651 {
5652 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5653 information before every VM-entry, hence disable it for performance sake. */
5654#if 0
5655 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5656 AssertRC(rc);
5657#endif
5658 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5659 }
5660
5661 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
5662 if ( fOffsettedTsc
5663 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5664 {
5665 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5666 {
5667 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, uTscOffset);
5668 AssertRC(rc);
5669 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5670 }
5671
5672 if (uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5673 {
5674 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5675 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5676 AssertRC(rc);
5677 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5678 }
5679 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5680 }
5681 else
5682 {
5683 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5684 if (!(uProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5685 {
5686 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5687 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
5688 AssertRC(rc);
5689 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
5690 }
5691 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5692 }
5693}
5694
5695
5696/**
5697 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5698 * VM-exit interruption info type.
5699 *
5700 * @returns The IEM exception flags.
5701 * @param uVector The event vector.
5702 * @param uVmxVectorType The VMX event type.
5703 *
5704 * @remarks This function currently only constructs flags required for
5705 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5706 * and CR2 aspects of an exception are not included).
5707 */
5708static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5709{
5710 uint32_t fIemXcptFlags;
5711 switch (uVmxVectorType)
5712 {
5713 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5714 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5715 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5716 break;
5717
5718 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5719 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5720 break;
5721
5722 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5723 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5724 break;
5725
5726 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5727 {
5728 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5729 if (uVector == X86_XCPT_BP)
5730 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5731 else if (uVector == X86_XCPT_OF)
5732 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5733 else
5734 {
5735 fIemXcptFlags = 0;
5736 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5737 }
5738 break;
5739 }
5740
5741 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5742 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5743 break;
5744
5745 default:
5746 fIemXcptFlags = 0;
5747 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5748 break;
5749 }
5750 return fIemXcptFlags;
5751}
5752
5753
5754/**
5755 * Sets an event as a pending event to be injected into the guest.
5756 *
5757 * @param pVCpu The cross context virtual CPU structure.
5758 * @param u32IntInfo The VM-entry interruption-information field.
5759 * @param cbInstr The VM-entry instruction length in bytes (for software
5760 * interrupts, exceptions and privileged software
5761 * exceptions).
5762 * @param u32ErrCode The VM-entry exception error code.
5763 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5764 * page-fault.
5765 *
5766 * @remarks Statistics counter assumes this is a guest event being injected or
5767 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5768 * always incremented.
5769 */
5770DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5771 RTGCUINTPTR GCPtrFaultAddress)
5772{
5773 Assert(!pVCpu->hm.s.Event.fPending);
5774 pVCpu->hm.s.Event.fPending = true;
5775 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5776 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5777 pVCpu->hm.s.Event.cbInstr = cbInstr;
5778 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5779}
5780
5781
5782/**
5783 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5784 *
5785 * @param pVCpu The cross context virtual CPU structure.
5786 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5787 * out-of-sync. Make sure to update the required fields
5788 * before using them.
5789 */
5790DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5791{
5792 NOREF(pMixedCtx);
5793 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5794 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5795 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5796 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5797}
5798
5799
5800/**
5801 * Handle a condition that occurred while delivering an event through the guest
5802 * IDT.
5803 *
5804 * @returns Strict VBox status code (i.e. informational status codes too).
5805 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5806 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5807 * to continue execution of the guest which will delivery the \#DF.
5808 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5809 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5810 *
5811 * @param pVCpu The cross context virtual CPU structure.
5812 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5813 * out-of-sync. Make sure to update the required fields
5814 * before using them.
5815 * @param pVmxTransient Pointer to the VMX transient structure.
5816 *
5817 * @remarks No-long-jump zone!!!
5818 */
5819static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5820{
5821 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5822
5823 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5824 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5825 AssertRCReturn(rc2, rc2);
5826
5827 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5828 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5829 {
5830 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5831 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5832
5833 /*
5834 * If the event was a software interrupt (generated with INT n) or a software exception
5835 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5836 * can handle the VM-exit and continue guest execution which will re-execute the
5837 * instruction rather than re-injecting the exception, as that can cause premature
5838 * trips to ring-3 before injection and involve TRPM which currently has no way of
5839 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5840 * the problem).
5841 */
5842 IEMXCPTRAISE enmRaise;
5843 IEMXCPTRAISEINFO fRaiseInfo;
5844 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5845 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5846 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5847 {
5848 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5849 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5850 }
5851 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5852 {
5853 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5854 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5855 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5856 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5857 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5858 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5859 uExitVectorType), VERR_VMX_IPE_5);
5860
5861 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5862
5863 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5864 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5865 {
5866 pVmxTransient->fVectoringPF = true;
5867 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5868 }
5869 }
5870 else
5871 {
5872 /*
5873 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5874 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5875 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5876 */
5877 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5878 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5879 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5880 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5881 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5882 }
5883
5884 /*
5885 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5886 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5887 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5888 * subsequent VM-entry would fail.
5889 *
5890 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5891 */
5892 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5893 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5894 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5895 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5896 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5897 {
5898 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5899 }
5900
5901 switch (enmRaise)
5902 {
5903 case IEMXCPTRAISE_CURRENT_XCPT:
5904 {
5905 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5906 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5907 Assert(rcStrict == VINF_SUCCESS);
5908 break;
5909 }
5910
5911 case IEMXCPTRAISE_PREV_EVENT:
5912 {
5913 uint32_t u32ErrCode;
5914 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5915 {
5916 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5917 AssertRCReturn(rc2, rc2);
5918 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5919 }
5920 else
5921 u32ErrCode = 0;
5922
5923 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5924 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5925 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5926 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5927
5928 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5929 pVCpu->hm.s.Event.u32ErrCode));
5930 Assert(rcStrict == VINF_SUCCESS);
5931 break;
5932 }
5933
5934 case IEMXCPTRAISE_REEXEC_INSTR:
5935 Assert(rcStrict == VINF_SUCCESS);
5936 break;
5937
5938 case IEMXCPTRAISE_DOUBLE_FAULT:
5939 {
5940 /*
5941 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5942 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5943 */
5944 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5945 {
5946 pVmxTransient->fVectoringDoublePF = true;
5947 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5948 pMixedCtx->cr2));
5949 rcStrict = VINF_SUCCESS;
5950 }
5951 else
5952 {
5953 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5954 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5955 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5956 uIdtVector, uExitVector));
5957 rcStrict = VINF_HM_DOUBLE_FAULT;
5958 }
5959 break;
5960 }
5961
5962 case IEMXCPTRAISE_TRIPLE_FAULT:
5963 {
5964 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5965 rcStrict = VINF_EM_RESET;
5966 break;
5967 }
5968
5969 case IEMXCPTRAISE_CPU_HANG:
5970 {
5971 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5972 rcStrict = VERR_EM_GUEST_CPU_HANG;
5973 break;
5974 }
5975
5976 default:
5977 {
5978 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5979 rcStrict = VERR_VMX_IPE_2;
5980 break;
5981 }
5982 }
5983 }
5984 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5985 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5986 && uExitVector != X86_XCPT_DF
5987 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5988 {
5989 /*
5990 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5991 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5992 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5993 */
5994 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5995 {
5996 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5997 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5998 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5999 }
6000 }
6001
6002 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
6003 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
6004 return rcStrict;
6005}
6006
6007
6008/**
6009 * Imports a guest segment register from the current VMCS into
6010 * the guest-CPU context.
6011 *
6012 * @returns VBox status code.
6013 * @param pVCpu The cross context virtual CPU structure.
6014 * @param idxSel Index of the selector in the VMCS.
6015 * @param idxLimit Index of the segment limit in the VMCS.
6016 * @param idxBase Index of the segment base in the VMCS.
6017 * @param idxAccess Index of the access rights of the segment in the VMCS.
6018 * @param pSelReg Pointer to the segment selector.
6019 *
6020 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6021 * do not log!
6022 *
6023 * @remarks Never call this function directly!!! Use the
6024 * HMVMX_IMPORT_SREG() macro as that takes care
6025 * of whether to read from the VMCS cache or not.
6026 */
6027static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
6028 PCPUMSELREG pSelReg)
6029{
6030 NOREF(pVCpu);
6031
6032 uint32_t u32Sel;
6033 uint32_t u32Limit;
6034 uint32_t u32Attr;
6035 uint64_t u64Base;
6036 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6037 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6038 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6039 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6040 AssertRCReturn(rc, rc);
6041
6042 pSelReg->Sel = (uint16_t)u32Sel;
6043 pSelReg->ValidSel = (uint16_t)u32Sel;
6044 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6045 pSelReg->u32Limit = u32Limit;
6046 pSelReg->u64Base = u64Base;
6047 pSelReg->Attr.u = u32Attr;
6048
6049 /*
6050 * If VT-x marks the segment as unusable, most other bits remain undefined:
6051 * - For CS the L, D and G bits have meaning.
6052 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6053 * - For the remaining data segments no bits are defined.
6054 *
6055 * The present bit and the unusable bit has been observed to be set at the
6056 * same time (the selector was supposed to be invalid as we started executing
6057 * a V8086 interrupt in ring-0).
6058 *
6059 * What should be important for the rest of the VBox code, is that the P bit is
6060 * cleared. Some of the other VBox code recognizes the unusable bit, but
6061 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6062 * safe side here, we'll strip off P and other bits we don't care about. If
6063 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6064 *
6065 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6066 */
6067 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6068 {
6069 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6070
6071 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6072 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6073 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6074#ifdef VBOX_STRICT
6075 VMMRZCallRing3Disable(pVCpu);
6076 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6077# ifdef DEBUG_bird
6078 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6079 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6080 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6081# endif
6082 VMMRZCallRing3Enable(pVCpu);
6083#endif
6084 }
6085 return VINF_SUCCESS;
6086}
6087
6088
6089/**
6090 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6091 *
6092 * @returns VBox status code.
6093 * @param pVCpu The cross context virtual CPU structure.
6094 *
6095 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6096 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6097 * instead!!!
6098 */
6099DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6100{
6101 uint64_t u64Val;
6102 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6103 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6104 {
6105 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6106 if (RT_SUCCESS(rc))
6107 {
6108 pCtx->rip = u64Val;
6109 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6110 }
6111 return rc;
6112 }
6113 return VINF_SUCCESS;
6114}
6115
6116
6117/**
6118 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6119 *
6120 * @returns VBox status code.
6121 * @param pVCpu The cross context virtual CPU structure.
6122 *
6123 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6124 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6125 * instead!!!
6126 */
6127DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6128{
6129 uint32_t u32Val;
6130 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6131 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6132 {
6133 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6134 if (RT_SUCCESS(rc))
6135 {
6136 pCtx->eflags.u32 = u32Val;
6137
6138 /* Restore eflags for real-on-v86-mode hack. */
6139 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6140 {
6141 pCtx->eflags.Bits.u1VM = 0;
6142 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6143 }
6144 }
6145 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6146 return rc;
6147 }
6148 return VINF_SUCCESS;
6149}
6150
6151
6152/**
6153 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6154 * context.
6155 *
6156 * @returns VBox status code.
6157 * @param pVCpu The cross context virtual CPU structure.
6158 *
6159 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6160 * do not log!
6161 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6162 * instead!!!
6163 */
6164DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6165{
6166 uint32_t u32Val;
6167 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6168 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6169 if (RT_SUCCESS(rc))
6170 {
6171 /*
6172 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6173 * might need them in hmR0VmxEvaluatePendingEvent().
6174 */
6175 if (!u32Val)
6176 {
6177 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6178 {
6179 rc = hmR0VmxImportGuestRip(pVCpu);
6180 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6181 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6182 }
6183
6184 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6185 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6186 }
6187 else
6188 {
6189 rc = hmR0VmxImportGuestRip(pVCpu);
6190 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6191
6192 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6193 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6194 {
6195 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6196 }
6197 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6198 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6199
6200 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6201 {
6202 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6203 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6204 }
6205 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6206 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6207 }
6208 }
6209 return rc;
6210}
6211
6212
6213/**
6214 * Worker for VMXR0ImportStateOnDemand.
6215 *
6216 * @returns VBox status code.
6217 * @param pVCpu The cross context virtual CPU structure.
6218 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6219 */
6220static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6221{
6222#define VMXLOCAL_BREAK_RC(a_rc) \
6223 if (RT_FAILURE(a_rc)) \
6224 break
6225
6226 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6227
6228 int rc = VINF_SUCCESS;
6229 PVM pVM = pVCpu->CTX_SUFF(pVM);
6230 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6231 uint64_t u64Val;
6232 uint32_t u32Val;
6233
6234 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6235
6236 /*
6237 * We disable interrupts to make the updating of the state and in particular
6238 * the fExtrn modification atomic wrt to preemption hooks.
6239 */
6240 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6241
6242 fWhat &= pCtx->fExtrn;
6243 if (fWhat)
6244 {
6245 do
6246 {
6247 if (fWhat & CPUMCTX_EXTRN_RIP)
6248 {
6249 rc = hmR0VmxImportGuestRip(pVCpu);
6250 VMXLOCAL_BREAK_RC(rc);
6251 }
6252
6253 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6254 {
6255 rc = hmR0VmxImportGuestRFlags(pVCpu);
6256 VMXLOCAL_BREAK_RC(rc);
6257 }
6258
6259 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6260 {
6261 rc = hmR0VmxImportGuestIntrState(pVCpu);
6262 VMXLOCAL_BREAK_RC(rc);
6263 }
6264
6265 if (fWhat & CPUMCTX_EXTRN_RSP)
6266 {
6267 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6268 VMXLOCAL_BREAK_RC(rc);
6269 pCtx->rsp = u64Val;
6270 }
6271
6272 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6273 {
6274 if (fWhat & CPUMCTX_EXTRN_CS)
6275 {
6276 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6277 VMXLOCAL_BREAK_RC(rc);
6278 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6279 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6280 }
6281 if (fWhat & CPUMCTX_EXTRN_SS)
6282 {
6283 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6284 VMXLOCAL_BREAK_RC(rc);
6285 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6286 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6287 }
6288 if (fWhat & CPUMCTX_EXTRN_DS)
6289 {
6290 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6291 VMXLOCAL_BREAK_RC(rc);
6292 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6293 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6294 }
6295 if (fWhat & CPUMCTX_EXTRN_ES)
6296 {
6297 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6298 VMXLOCAL_BREAK_RC(rc);
6299 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6300 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6301 }
6302 if (fWhat & CPUMCTX_EXTRN_FS)
6303 {
6304 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6305 VMXLOCAL_BREAK_RC(rc);
6306 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6307 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6308 }
6309 if (fWhat & CPUMCTX_EXTRN_GS)
6310 {
6311 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6312 VMXLOCAL_BREAK_RC(rc);
6313 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6314 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6315 }
6316 }
6317
6318 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6319 {
6320 if (fWhat & CPUMCTX_EXTRN_LDTR)
6321 {
6322 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6323 VMXLOCAL_BREAK_RC(rc);
6324 }
6325
6326 if (fWhat & CPUMCTX_EXTRN_GDTR)
6327 {
6328 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6329 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6330 VMXLOCAL_BREAK_RC(rc);
6331 pCtx->gdtr.pGdt = u64Val;
6332 pCtx->gdtr.cbGdt = u32Val;
6333 }
6334
6335 /* Guest IDTR. */
6336 if (fWhat & CPUMCTX_EXTRN_IDTR)
6337 {
6338 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6339 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6340 VMXLOCAL_BREAK_RC(rc);
6341 pCtx->idtr.pIdt = u64Val;
6342 pCtx->idtr.cbIdt = u32Val;
6343 }
6344
6345 /* Guest TR. */
6346 if (fWhat & CPUMCTX_EXTRN_TR)
6347 {
6348 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6349 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6350 {
6351 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6352 VMXLOCAL_BREAK_RC(rc);
6353 }
6354 }
6355 }
6356
6357 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6358 {
6359 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6360 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6361 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6362 pCtx->SysEnter.cs = u32Val;
6363 VMXLOCAL_BREAK_RC(rc);
6364 }
6365
6366#if HC_ARCH_BITS == 64
6367 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6368 {
6369 if ( pVM->hm.s.fAllow64BitGuests
6370 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6371 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6372 }
6373
6374 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6375 {
6376 if ( pVM->hm.s.fAllow64BitGuests
6377 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6378 {
6379 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6380 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6381 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6382 }
6383 }
6384#endif
6385
6386 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6387#if HC_ARCH_BITS == 32
6388 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6389#endif
6390 )
6391 {
6392 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6393 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6394 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6395 {
6396 switch (pMsr->u32Msr)
6397 {
6398#if HC_ARCH_BITS == 32
6399 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6400 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6401 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6402 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6403#endif
6404 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6405 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6406 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6407 default:
6408 {
6409 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6410 ASMSetFlags(fEFlags);
6411 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6412 cMsrs));
6413 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6414 }
6415 }
6416 }
6417 }
6418
6419 if (fWhat & CPUMCTX_EXTRN_DR7)
6420 {
6421 if (!pVCpu->hm.s.fUsingHyperDR7)
6422 {
6423 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6424 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6425 VMXLOCAL_BREAK_RC(rc);
6426 pCtx->dr[7] = u32Val;
6427 }
6428 }
6429
6430 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6431 {
6432 uint32_t u32Shadow;
6433 if (fWhat & CPUMCTX_EXTRN_CR0)
6434 {
6435 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6436 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6437 VMXLOCAL_BREAK_RC(rc);
6438 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr0Mask)
6439 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr0Mask);
6440 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6441 CPUMSetGuestCR0(pVCpu, u32Val);
6442 VMMRZCallRing3Enable(pVCpu);
6443 }
6444
6445 if (fWhat & CPUMCTX_EXTRN_CR4)
6446 {
6447 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6448 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6449 VMXLOCAL_BREAK_RC(rc);
6450 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32Cr4Mask)
6451 | (u32Shadow & pVCpu->hm.s.vmx.u32Cr4Mask);
6452 CPUMSetGuestCR4(pVCpu, u32Val);
6453 }
6454
6455 if (fWhat & CPUMCTX_EXTRN_CR3)
6456 {
6457 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6458 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6459 || ( pVM->hm.s.fNestedPaging
6460 && CPUMIsGuestPagingEnabledEx(pCtx)))
6461 {
6462 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6463 if (pCtx->cr3 != u64Val)
6464 {
6465 CPUMSetGuestCR3(pVCpu, u64Val);
6466 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6467 }
6468
6469 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6470 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6471 if (CPUMIsGuestInPAEModeEx(pCtx))
6472 {
6473 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6474 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6475 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6476 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6477 VMXLOCAL_BREAK_RC(rc);
6478 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6479 }
6480 }
6481 }
6482 }
6483 } while (0);
6484
6485 if (RT_SUCCESS(rc))
6486 {
6487 /* Update fExtrn. */
6488 pCtx->fExtrn &= ~fWhat;
6489
6490 /* If everything has been imported, clear the HM keeper bit. */
6491 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6492 {
6493 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6494 Assert(!pCtx->fExtrn);
6495 }
6496 }
6497 }
6498 else
6499 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6500
6501 ASMSetFlags(fEFlags);
6502
6503 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6504
6505 /*
6506 * Honor any pending CR3 updates.
6507 *
6508 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6509 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6510 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6511 *
6512 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6513 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6514 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6515 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6516 *
6517 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6518 */
6519 if (VMMRZCallRing3IsEnabled(pVCpu))
6520 {
6521 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6522 {
6523 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6524 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6525 }
6526
6527 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6528 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6529
6530 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6531 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6532 }
6533
6534 return VINF_SUCCESS;
6535#undef VMXLOCAL_BREAK_RC
6536}
6537
6538
6539/**
6540 * Saves the guest state from the VMCS into the guest-CPU context.
6541 *
6542 * @returns VBox status code.
6543 * @param pVCpu The cross context virtual CPU structure.
6544 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6545 */
6546VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6547{
6548 return hmR0VmxImportGuestState(pVCpu, fWhat);
6549}
6550
6551
6552/**
6553 * Check per-VM and per-VCPU force flag actions that require us to go back to
6554 * ring-3 for one reason or another.
6555 *
6556 * @returns Strict VBox status code (i.e. informational status codes too)
6557 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6558 * ring-3.
6559 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6560 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6561 * interrupts)
6562 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6563 * all EMTs to be in ring-3.
6564 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6565 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6566 * to the EM loop.
6567 *
6568 * @param pVCpu The cross context virtual CPU structure.
6569 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6570 * out-of-sync. Make sure to update the required fields
6571 * before using them.
6572 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6573 */
6574static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6575{
6576 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6577
6578 /*
6579 * Anything pending? Should be more likely than not if we're doing a good job.
6580 */
6581 PVM pVM = pVCpu->CTX_SUFF(pVM);
6582 if ( !fStepping
6583 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6584 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6585 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6586 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6587 return VINF_SUCCESS;
6588
6589 /* Pending PGM C3 sync. */
6590 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6591 {
6592 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6593 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6594 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6595 if (rcStrict2 != VINF_SUCCESS)
6596 {
6597 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6598 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6599 return rcStrict2;
6600 }
6601 }
6602
6603 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6604 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6605 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6606 {
6607 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6608 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6609 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6610 return rc2;
6611 }
6612
6613 /* Pending VM request packets, such as hardware interrupts. */
6614 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6615 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6616 {
6617 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6618 return VINF_EM_PENDING_REQUEST;
6619 }
6620
6621 /* Pending PGM pool flushes. */
6622 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6623 {
6624 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6625 return VINF_PGM_POOL_FLUSH_PENDING;
6626 }
6627
6628 /* Pending DMA requests. */
6629 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6630 {
6631 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6632 return VINF_EM_RAW_TO_R3;
6633 }
6634
6635 return VINF_SUCCESS;
6636}
6637
6638
6639/**
6640 * Converts any TRPM trap into a pending HM event. This is typically used when
6641 * entering from ring-3 (not longjmp returns).
6642 *
6643 * @param pVCpu The cross context virtual CPU structure.
6644 */
6645static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6646{
6647 Assert(TRPMHasTrap(pVCpu));
6648 Assert(!pVCpu->hm.s.Event.fPending);
6649
6650 uint8_t uVector;
6651 TRPMEVENT enmTrpmEvent;
6652 RTGCUINT uErrCode;
6653 RTGCUINTPTR GCPtrFaultAddress;
6654 uint8_t cbInstr;
6655
6656 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6657 AssertRC(rc);
6658
6659 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6660 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6661 if (enmTrpmEvent == TRPM_TRAP)
6662 {
6663 switch (uVector)
6664 {
6665 case X86_XCPT_NMI:
6666 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6667 break;
6668
6669 case X86_XCPT_BP:
6670 case X86_XCPT_OF:
6671 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6672 break;
6673
6674 case X86_XCPT_PF:
6675 case X86_XCPT_DF:
6676 case X86_XCPT_TS:
6677 case X86_XCPT_NP:
6678 case X86_XCPT_SS:
6679 case X86_XCPT_GP:
6680 case X86_XCPT_AC:
6681 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6682 RT_FALL_THRU();
6683 default:
6684 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6685 break;
6686 }
6687 }
6688 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6689 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6690 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6691 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6692 else
6693 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6694
6695 rc = TRPMResetTrap(pVCpu);
6696 AssertRC(rc);
6697 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6698 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6699
6700 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6701}
6702
6703
6704/**
6705 * Converts the pending HM event into a TRPM trap.
6706 *
6707 * @param pVCpu The cross context virtual CPU structure.
6708 */
6709static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6710{
6711 Assert(pVCpu->hm.s.Event.fPending);
6712
6713 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6714 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6715 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6716 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6717
6718 /* If a trap was already pending, we did something wrong! */
6719 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6720
6721 TRPMEVENT enmTrapType;
6722 switch (uVectorType)
6723 {
6724 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6725 enmTrapType = TRPM_HARDWARE_INT;
6726 break;
6727
6728 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6729 enmTrapType = TRPM_SOFTWARE_INT;
6730 break;
6731
6732 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6733 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6734 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6735 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6736 enmTrapType = TRPM_TRAP;
6737 break;
6738
6739 default:
6740 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6741 enmTrapType = TRPM_32BIT_HACK;
6742 break;
6743 }
6744
6745 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6746
6747 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6748 AssertRC(rc);
6749
6750 if (fErrorCodeValid)
6751 TRPMSetErrorCode(pVCpu, uErrorCode);
6752
6753 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6754 && uVector == X86_XCPT_PF)
6755 {
6756 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6757 }
6758 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6759 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6760 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6761 {
6762 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6763 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6764 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6765 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6766 }
6767
6768 /* Clear any pending events from the VMCS. */
6769 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6770 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6771
6772 /* We're now done converting the pending event. */
6773 pVCpu->hm.s.Event.fPending = false;
6774}
6775
6776
6777/**
6778 * Does the necessary state syncing before returning to ring-3 for any reason
6779 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6780 *
6781 * @returns VBox status code.
6782 * @param pVCpu The cross context virtual CPU structure.
6783 * @param fImportState Whether to import the guest state from the VMCS back
6784 * to the guest-CPU context.
6785 *
6786 * @remarks No-long-jmp zone!!!
6787 */
6788static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6789{
6790 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6791 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6792
6793 RTCPUID idCpu = RTMpCpuId();
6794 Log4Func(("HostCpuId=%u\n", idCpu));
6795
6796 /*
6797 * !!! IMPORTANT !!!
6798 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6799 */
6800
6801 /* Save the guest state if necessary. */
6802 if (fImportState)
6803 {
6804 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6805 AssertRCReturn(rc, rc);
6806 }
6807
6808 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6809 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6810 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6811
6812 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6813#ifdef VBOX_STRICT
6814 if (CPUMIsHyperDebugStateActive(pVCpu))
6815 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6816#endif
6817 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6818 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6819 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6820
6821#if HC_ARCH_BITS == 64
6822 /* Restore host-state bits that VT-x only restores partially. */
6823 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6824 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6825 {
6826 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6827 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6828 }
6829 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6830#endif
6831
6832 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6833 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6834 {
6835 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6836 if (!fImportState)
6837 {
6838 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6839 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6840 AssertRCReturn(rc, rc);
6841 }
6842 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6843 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6844 }
6845 else
6846 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6847
6848 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6849 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6850
6851 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6852 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6853 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6854 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6855 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6856 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6857 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6858 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6859 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6860
6861 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6862
6863 /** @todo This partially defeats the purpose of having preemption hooks.
6864 * The problem is, deregistering the hooks should be moved to a place that
6865 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6866 * context.
6867 */
6868 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6869 {
6870 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6871 AssertRCReturn(rc, rc);
6872
6873 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6874 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6875 }
6876 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6877 NOREF(idCpu);
6878
6879 return VINF_SUCCESS;
6880}
6881
6882
6883/**
6884 * Leaves the VT-x session.
6885 *
6886 * @returns VBox status code.
6887 * @param pVCpu The cross context virtual CPU structure.
6888 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6889 * out-of-sync. Make sure to update the required fields
6890 * before using them.
6891 *
6892 * @remarks No-long-jmp zone!!!
6893 */
6894static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6895{
6896 HM_DISABLE_PREEMPT();
6897 HMVMX_ASSERT_CPU_SAFE();
6898 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6899 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6900
6901 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6902 and done this from the VMXR0ThreadCtxCallback(). */
6903 if (!pVCpu->hm.s.fLeaveDone)
6904 {
6905 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6906 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6907 pVCpu->hm.s.fLeaveDone = true;
6908 }
6909 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6910
6911 /*
6912 * !!! IMPORTANT !!!
6913 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6914 */
6915
6916 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6917 /** @todo Deregistering here means we need to VMCLEAR always
6918 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6919 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6920 VMMR0ThreadCtxHookDisable(pVCpu);
6921
6922 /* Leave HM context. This takes care of local init (term). */
6923 int rc = HMR0LeaveCpu(pVCpu);
6924
6925 HM_RESTORE_PREEMPT();
6926 return rc;
6927}
6928
6929
6930/**
6931 * Does the necessary state syncing before doing a longjmp to ring-3.
6932 *
6933 * @returns VBox status code.
6934 * @param pVCpu The cross context virtual CPU structure.
6935 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6936 * out-of-sync. Make sure to update the required fields
6937 * before using them.
6938 *
6939 * @remarks No-long-jmp zone!!!
6940 */
6941DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6942{
6943 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6944}
6945
6946
6947/**
6948 * Take necessary actions before going back to ring-3.
6949 *
6950 * An action requires us to go back to ring-3. This function does the necessary
6951 * steps before we can safely return to ring-3. This is not the same as longjmps
6952 * to ring-3, this is voluntary and prepares the guest so it may continue
6953 * executing outside HM (recompiler/IEM).
6954 *
6955 * @returns VBox status code.
6956 * @param pVCpu The cross context virtual CPU structure.
6957 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6958 * out-of-sync. Make sure to update the required fields
6959 * before using them.
6960 * @param rcExit The reason for exiting to ring-3. Can be
6961 * VINF_VMM_UNKNOWN_RING3_CALL.
6962 */
6963static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6964{
6965 Assert(pVCpu);
6966 Assert(pMixedCtx);
6967 HMVMX_ASSERT_PREEMPT_SAFE();
6968
6969 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6970 {
6971 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6972 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6973 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6974 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6975 }
6976
6977 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6978 VMMRZCallRing3Disable(pVCpu);
6979 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6980
6981 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6982 if (pVCpu->hm.s.Event.fPending)
6983 {
6984 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6985 Assert(!pVCpu->hm.s.Event.fPending);
6986 }
6987
6988 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6989 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6990
6991 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6992 and if we're injecting an event we should have a TRPM trap pending. */
6993 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6994#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6995 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6996#endif
6997
6998 /* Save guest state and restore host state bits. */
6999 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
7000 AssertRCReturn(rc, rc);
7001 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
7002 /* Thread-context hooks are unregistered at this point!!! */
7003
7004 /* Sync recompiler state. */
7005 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
7006 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
7007 | CPUM_CHANGED_LDTR
7008 | CPUM_CHANGED_GDTR
7009 | CPUM_CHANGED_IDTR
7010 | CPUM_CHANGED_TR
7011 | CPUM_CHANGED_HIDDEN_SEL_REGS);
7012 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
7013 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
7014 {
7015 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
7016 }
7017
7018 Assert(!pVCpu->hm.s.fClearTrapFlag);
7019
7020 /* Update the exit-to-ring 3 reason. */
7021 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
7022
7023 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
7024 if (rcExit != VINF_EM_RAW_INTERRUPT)
7025 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
7026
7027 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
7028
7029 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
7030 VMMRZCallRing3RemoveNotification(pVCpu);
7031 VMMRZCallRing3Enable(pVCpu);
7032
7033 return rc;
7034}
7035
7036
7037/**
7038 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7039 * longjump to ring-3 and possibly get preempted.
7040 *
7041 * @returns VBox status code.
7042 * @param pVCpu The cross context virtual CPU structure.
7043 * @param enmOperation The operation causing the ring-3 longjump.
7044 * @param pvUser Opaque pointer to the guest-CPU context. The data
7045 * may be out-of-sync. Make sure to update the required
7046 * fields before using them.
7047 */
7048static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7049{
7050 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7051 {
7052 /*
7053 * !!! IMPORTANT !!!
7054 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7055 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7056 */
7057 VMMRZCallRing3RemoveNotification(pVCpu);
7058 VMMRZCallRing3Disable(pVCpu);
7059 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7060 RTThreadPreemptDisable(&PreemptState);
7061
7062 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7063 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7064
7065#if HC_ARCH_BITS == 64
7066 /* Restore host-state bits that VT-x only restores partially. */
7067 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7068 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7069 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7070 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7071#endif
7072
7073 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7074 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7075 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7076
7077 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7078 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7079 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7080 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7081 {
7082 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7083 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7084 }
7085
7086 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7087 VMMR0ThreadCtxHookDisable(pVCpu);
7088 HMR0LeaveCpu(pVCpu);
7089 RTThreadPreemptRestore(&PreemptState);
7090 return VINF_SUCCESS;
7091 }
7092
7093 Assert(pVCpu);
7094 Assert(pvUser);
7095 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7096 HMVMX_ASSERT_PREEMPT_SAFE();
7097
7098 VMMRZCallRing3Disable(pVCpu);
7099 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7100
7101 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7102
7103 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7104 AssertRCReturn(rc, rc);
7105
7106 VMMRZCallRing3Enable(pVCpu);
7107 return VINF_SUCCESS;
7108}
7109
7110
7111/**
7112 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7113 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7114 *
7115 * @param pVCpu The cross context virtual CPU structure.
7116 */
7117DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7118{
7119 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7120 {
7121 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7122 {
7123 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7124 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7125 AssertRC(rc);
7126 Log4Func(("Setup interrupt-window exiting\n"));
7127 }
7128 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7129}
7130
7131
7132/**
7133 * Clears the interrupt-window exiting control in the VMCS.
7134 *
7135 * @param pVCpu The cross context virtual CPU structure.
7136 */
7137DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7138{
7139 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7140 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7141 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7142 AssertRC(rc);
7143 Log4Func(("Cleared interrupt-window exiting\n"));
7144}
7145
7146
7147/**
7148 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7149 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7150 *
7151 * @param pVCpu The cross context virtual CPU structure.
7152 */
7153DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7154{
7155 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7156 {
7157 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7158 {
7159 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7160 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7161 AssertRC(rc);
7162 Log4Func(("Setup NMI-window exiting\n"));
7163 }
7164 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7165}
7166
7167
7168/**
7169 * Clears the NMI-window exiting control in the VMCS.
7170 *
7171 * @param pVCpu The cross context virtual CPU structure.
7172 */
7173DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7174{
7175 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7176 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7177 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7178 AssertRC(rc);
7179 Log4Func(("Cleared NMI-window exiting\n"));
7180}
7181
7182
7183/**
7184 * Evaluates the event to be delivered to the guest and sets it as the pending
7185 * event.
7186 *
7187 * @returns The VT-x guest-interruptibility state.
7188 * @param pVCpu The cross context virtual CPU structure.
7189 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7190 * out-of-sync. Make sure to update the required fields
7191 * before using them.
7192 */
7193static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7194{
7195 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7196 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7197 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7198 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7199 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7200
7201 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7202 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7203 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7204 Assert(!TRPMHasTrap(pVCpu));
7205
7206 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7207 APICUpdatePendingInterrupts(pVCpu);
7208
7209 /*
7210 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7211 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7212 */
7213 /** @todo SMI. SMIs take priority over NMIs. */
7214 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7215 {
7216 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7217 if ( !pVCpu->hm.s.Event.fPending
7218 && !fBlockNmi
7219 && !fBlockSti
7220 && !fBlockMovSS)
7221 {
7222 Log4Func(("Pending NMI\n"));
7223 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7224 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7225
7226 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7227 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7228 }
7229 else
7230 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7231 }
7232 /*
7233 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7234 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7235 */
7236 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7237 && !pVCpu->hm.s.fSingleInstruction)
7238 {
7239 Assert(!DBGFIsStepping(pVCpu));
7240 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7241 AssertRCReturn(rc, 0);
7242 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7243 if ( !pVCpu->hm.s.Event.fPending
7244 && !fBlockInt
7245 && !fBlockSti
7246 && !fBlockMovSS)
7247 {
7248 uint8_t u8Interrupt;
7249 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7250 if (RT_SUCCESS(rc))
7251 {
7252 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7253 uint32_t u32IntInfo = u8Interrupt
7254 | VMX_EXIT_INTERRUPTION_INFO_VALID
7255 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7256
7257 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7258 }
7259 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7260 {
7261 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7262 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7263 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7264
7265 /*
7266 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7267 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7268 * need to re-set this force-flag here.
7269 */
7270 }
7271 else
7272 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7273 }
7274 else
7275 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7276 }
7277
7278 return fIntrState;
7279}
7280
7281
7282/**
7283 * Sets a pending-debug exception to be delivered to the guest if the guest is
7284 * single-stepping in the VMCS.
7285 *
7286 * @param pVCpu The cross context virtual CPU structure.
7287 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7288 * out-of-sync. Make sure to update the required fields
7289 * before using them.
7290 */
7291DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7292{
7293 RT_NOREF(pVCpu);
7294 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7295 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7296}
7297
7298
7299/**
7300 * Injects any pending events into the guest if the guest is in a state to
7301 * receive them.
7302 *
7303 * @returns Strict VBox status code (i.e. informational status codes too).
7304 * @param pVCpu The cross context virtual CPU structure.
7305 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7306 * out-of-sync. Make sure to update the required fields
7307 * before using them.
7308 * @param fIntrState The VT-x guest-interruptibility state.
7309 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7310 * return VINF_EM_DBG_STEPPED if the event was
7311 * dispatched directly.
7312 */
7313static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7314{
7315 HMVMX_ASSERT_PREEMPT_SAFE();
7316 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7317
7318 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7319 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7320
7321 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7322 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7323 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7324 Assert(!TRPMHasTrap(pVCpu));
7325
7326 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7327 if (pVCpu->hm.s.Event.fPending)
7328 {
7329 /*
7330 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7331 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7332 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7333 *
7334 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7335 */
7336 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7337#ifdef VBOX_STRICT
7338 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7339 {
7340 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7341 Assert(!fBlockInt);
7342 Assert(!fBlockSti);
7343 Assert(!fBlockMovSS);
7344 }
7345 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7346 {
7347 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7348 Assert(!fBlockSti);
7349 Assert(!fBlockMovSS);
7350 Assert(!fBlockNmi);
7351 }
7352#endif
7353 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7354 uIntType));
7355 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7356 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7357 &fIntrState);
7358 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7359
7360 /* Update the interruptibility-state as it could have been changed by
7361 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7362 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7363 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7364
7365 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7366 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7367 else
7368 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7369 }
7370
7371 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7372 if ( fBlockSti
7373 || fBlockMovSS)
7374 {
7375 if (!pVCpu->hm.s.fSingleInstruction)
7376 {
7377 /*
7378 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7379 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7380 * See Intel spec. 27.3.4 "Saving Non-Register State".
7381 */
7382 Assert(!DBGFIsStepping(pVCpu));
7383 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7384 AssertRCReturn(rc, rc);
7385 if (pMixedCtx->eflags.Bits.u1TF)
7386 {
7387 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7388 AssertRCReturn(rc2, rc2);
7389 }
7390 }
7391 else if (pMixedCtx->eflags.Bits.u1TF)
7392 {
7393 /*
7394 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7395 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7396 */
7397 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7398 fIntrState = 0;
7399 }
7400 }
7401
7402 /*
7403 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7404 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7405 */
7406 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7407 AssertRCReturn(rc3, rc3);
7408
7409 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7410 NOREF(fBlockMovSS); NOREF(fBlockSti);
7411 return rcStrict;
7412}
7413
7414
7415/**
7416 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7417 *
7418 * @param pVCpu The cross context virtual CPU structure.
7419 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7420 * out-of-sync. Make sure to update the required fields
7421 * before using them.
7422 */
7423DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7424{
7425 NOREF(pMixedCtx);
7426 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7427 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7428}
7429
7430
7431/**
7432 * Injects a double-fault (\#DF) exception into the VM.
7433 *
7434 * @returns Strict VBox status code (i.e. informational status codes too).
7435 * @param pVCpu The cross context virtual CPU structure.
7436 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7437 * out-of-sync. Make sure to update the required fields
7438 * before using them.
7439 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7440 * and should return VINF_EM_DBG_STEPPED if the event
7441 * is injected directly (register modified by us, not
7442 * by hardware on VM-entry).
7443 * @param pfIntrState Pointer to the current guest interruptibility-state.
7444 * This interruptibility-state will be updated if
7445 * necessary. This cannot not be NULL.
7446 */
7447DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7448{
7449 NOREF(pMixedCtx);
7450 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7451 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7452 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7453 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7454 pfIntrState);
7455}
7456
7457
7458/**
7459 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7460 *
7461 * @param pVCpu The cross context virtual CPU structure.
7462 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7463 * out-of-sync. Make sure to update the required fields
7464 * before using them.
7465 */
7466DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7467{
7468 NOREF(pMixedCtx);
7469 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7470 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7471 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7472}
7473
7474
7475/**
7476 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7477 *
7478 * @param pVCpu The cross context virtual CPU structure.
7479 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7480 * out-of-sync. Make sure to update the required fields
7481 * before using them.
7482 * @param cbInstr The value of RIP that is to be pushed on the guest
7483 * stack.
7484 */
7485DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7486{
7487 NOREF(pMixedCtx);
7488 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7489 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7490 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7491}
7492
7493
7494/**
7495 * Injects a general-protection (\#GP) fault into the VM.
7496 *
7497 * @returns Strict VBox status code (i.e. informational status codes too).
7498 * @param pVCpu The cross context virtual CPU structure.
7499 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7500 * out-of-sync. Make sure to update the required fields
7501 * before using them.
7502 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7503 * mode, i.e. in real-mode it's not valid).
7504 * @param u32ErrorCode The error code associated with the \#GP.
7505 * @param fStepping Whether we're running in
7506 * hmR0VmxRunGuestCodeStep() and should return
7507 * VINF_EM_DBG_STEPPED if the event is injected
7508 * directly (register modified by us, not by
7509 * hardware on VM-entry).
7510 * @param pfIntrState Pointer to the current guest interruptibility-state.
7511 * This interruptibility-state will be updated if
7512 * necessary. This cannot not be NULL.
7513 */
7514DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7515 bool fStepping, uint32_t *pfIntrState)
7516{
7517 NOREF(pMixedCtx);
7518 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7519 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7520 if (fErrorCodeValid)
7521 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7522 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7523 pfIntrState);
7524}
7525
7526
7527#if 0 /* unused */
7528/**
7529 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7530 * VM.
7531 *
7532 * @param pVCpu The cross context virtual CPU structure.
7533 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7534 * out-of-sync. Make sure to update the required fields
7535 * before using them.
7536 * @param u32ErrorCode The error code associated with the \#GP.
7537 */
7538DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7539{
7540 NOREF(pMixedCtx);
7541 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7542 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7543 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7544 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7545}
7546#endif /* unused */
7547
7548
7549/**
7550 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7551 *
7552 * @param pVCpu The cross context virtual CPU structure.
7553 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7554 * out-of-sync. Make sure to update the required fields
7555 * before using them.
7556 * @param uVector The software interrupt vector number.
7557 * @param cbInstr The value of RIP that is to be pushed on the guest
7558 * stack.
7559 */
7560DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7561{
7562 NOREF(pMixedCtx);
7563 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7564 if ( uVector == X86_XCPT_BP
7565 || uVector == X86_XCPT_OF)
7566 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7567 else
7568 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7569 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7570}
7571
7572
7573/**
7574 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7575 * stack.
7576 *
7577 * @returns Strict VBox status code (i.e. informational status codes too).
7578 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7579 * @param pVM The cross context VM structure.
7580 * @param pMixedCtx Pointer to the guest-CPU context.
7581 * @param uValue The value to push to the guest stack.
7582 */
7583DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7584{
7585 /*
7586 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7587 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7588 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7589 */
7590 if (pMixedCtx->sp == 1)
7591 return VINF_EM_RESET;
7592 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7593 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7594 AssertRC(rc);
7595 return rc;
7596}
7597
7598
7599/**
7600 * Injects an event into the guest upon VM-entry by updating the relevant fields
7601 * in the VM-entry area in the VMCS.
7602 *
7603 * @returns Strict VBox status code (i.e. informational status codes too).
7604 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7605 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7606 *
7607 * @param pVCpu The cross context virtual CPU structure.
7608 * @param u64IntInfo The VM-entry interruption-information field.
7609 * @param cbInstr The VM-entry instruction length in bytes (for
7610 * software interrupts, exceptions and privileged
7611 * software exceptions).
7612 * @param u32ErrCode The VM-entry exception error code.
7613 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7614 * @param pfIntrState Pointer to the current guest interruptibility-state.
7615 * This interruptibility-state will be updated if
7616 * necessary. This cannot not be NULL.
7617 * @param fStepping Whether we're running in
7618 * hmR0VmxRunGuestCodeStep() and should return
7619 * VINF_EM_DBG_STEPPED if the event is injected
7620 * directly (register modified by us, not by
7621 * hardware on VM-entry).
7622 */
7623static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7624 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7625{
7626 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7627 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7628 Assert(pfIntrState);
7629
7630 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7631 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7632 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7633 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7634
7635#ifdef VBOX_STRICT
7636 /*
7637 * Validate the error-code-valid bit for hardware exceptions.
7638 * No error codes for exceptions in real-mode.
7639 *
7640 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7641 */
7642 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7643 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7644 {
7645 switch (uVector)
7646 {
7647 case X86_XCPT_PF:
7648 case X86_XCPT_DF:
7649 case X86_XCPT_TS:
7650 case X86_XCPT_NP:
7651 case X86_XCPT_SS:
7652 case X86_XCPT_GP:
7653 case X86_XCPT_AC:
7654 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7655 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7656 RT_FALL_THRU();
7657 default:
7658 break;
7659 }
7660 }
7661#endif
7662
7663 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7664 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7665 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7666
7667 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7668
7669 /*
7670 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7671 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7672 * interrupt handler in the (real-mode) guest.
7673 *
7674 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7675 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7676 */
7677 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7678 {
7679 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7680 {
7681 /*
7682 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7683 * set the deliver-error-code bit.
7684 *
7685 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7686 */
7687 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7688 }
7689 else
7690 {
7691 PVM pVM = pVCpu->CTX_SUFF(pVM);
7692 Assert(PDMVmmDevHeapIsEnabled(pVM));
7693 Assert(pVM->hm.s.vmx.pRealModeTSS);
7694
7695 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7696 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7697 | CPUMCTX_EXTRN_TABLE_MASK
7698 | CPUMCTX_EXTRN_RIP
7699 | CPUMCTX_EXTRN_RSP
7700 | CPUMCTX_EXTRN_RFLAGS);
7701 AssertRCReturn(rc2, rc2);
7702
7703 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7704 size_t const cbIdtEntry = sizeof(X86IDTR16);
7705 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7706 {
7707 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7708 if (uVector == X86_XCPT_DF)
7709 return VINF_EM_RESET;
7710
7711 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7712 if (uVector == X86_XCPT_GP)
7713 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7714
7715 /*
7716 * If we're injecting an event with no valid IDT entry, inject a #GP.
7717 * No error codes for exceptions in real-mode.
7718 *
7719 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7720 */
7721 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7722 pfIntrState);
7723 }
7724
7725 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7726 uint16_t uGuestIp = pMixedCtx->ip;
7727 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7728 {
7729 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7730 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7731 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7732 }
7733 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7734 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7735
7736 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7737 X86IDTR16 IdtEntry;
7738 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7739 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7740 AssertRCReturn(rc2, rc2);
7741
7742 /* Construct the stack frame for the interrupt/exception handler. */
7743 VBOXSTRICTRC rcStrict;
7744 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7745 if (rcStrict == VINF_SUCCESS)
7746 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7747 if (rcStrict == VINF_SUCCESS)
7748 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7749
7750 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7751 if (rcStrict == VINF_SUCCESS)
7752 {
7753 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7754 pMixedCtx->rip = IdtEntry.offSel;
7755 pMixedCtx->cs.Sel = IdtEntry.uSel;
7756 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7757 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7758 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7759 && uVector == X86_XCPT_PF)
7760 pMixedCtx->cr2 = GCPtrFaultAddress;
7761
7762 /* If any other guest-state bits are changed here, make sure to update
7763 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7764 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7765 | HM_CHANGED_GUEST_CR2
7766 | HM_CHANGED_GUEST_RIP
7767 | HM_CHANGED_GUEST_RFLAGS
7768 | HM_CHANGED_GUEST_RSP);
7769
7770 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7771 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7772 {
7773 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7774 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7775 Log4Func(("Clearing inhibition due to STI\n"));
7776 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7777 }
7778 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7779 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7780
7781 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7782 it, if we are returning to ring-3 before executing guest code. */
7783 pVCpu->hm.s.Event.fPending = false;
7784
7785 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7786 if (fStepping)
7787 rcStrict = VINF_EM_DBG_STEPPED;
7788 }
7789 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7790 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7791 return rcStrict;
7792 }
7793 }
7794
7795 /* Validate. */
7796 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7797 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7798 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7799
7800 /* Inject. */
7801 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7802 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7803 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7804 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7805 AssertRCReturn(rc, rc);
7806
7807 /* Update CR2. */
7808 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7809 && uVector == X86_XCPT_PF)
7810 pMixedCtx->cr2 = GCPtrFaultAddress;
7811
7812 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7813
7814 return VINF_SUCCESS;
7815}
7816
7817
7818/**
7819 * Clears the interrupt-window exiting control in the VMCS and if necessary
7820 * clears the current event in the VMCS as well.
7821 *
7822 * @returns VBox status code.
7823 * @param pVCpu The cross context virtual CPU structure.
7824 *
7825 * @remarks Use this function only to clear events that have not yet been
7826 * delivered to the guest but are injected in the VMCS!
7827 * @remarks No-long-jump zone!!!
7828 */
7829static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7830{
7831 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7832 {
7833 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7834 Log4Func(("Cleared interrupt widow\n"));
7835 }
7836
7837 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7838 {
7839 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7840 Log4Func(("Cleared interrupt widow\n"));
7841 }
7842}
7843
7844
7845/**
7846 * Enters the VT-x session.
7847 *
7848 * @returns VBox status code.
7849 * @param pVCpu The cross context virtual CPU structure.
7850 * @param pHostCpu Pointer to the global CPU info struct.
7851 */
7852VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7853{
7854 AssertPtr(pVCpu);
7855 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7856 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7857 RT_NOREF(pHostCpu);
7858
7859 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7860 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7861 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7862
7863#ifdef VBOX_STRICT
7864 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7865 RTCCUINTREG uHostCR4 = ASMGetCR4();
7866 if (!(uHostCR4 & X86_CR4_VMXE))
7867 {
7868 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7869 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7870 }
7871#endif
7872
7873 /*
7874 * Load the VCPU's VMCS as the current (and active) one.
7875 */
7876 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7877 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7878 if (RT_FAILURE(rc))
7879 return rc;
7880
7881 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7882 pVCpu->hm.s.fLeaveDone = false;
7883 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7884
7885 return VINF_SUCCESS;
7886}
7887
7888
7889/**
7890 * The thread-context callback (only on platforms which support it).
7891 *
7892 * @param enmEvent The thread-context event.
7893 * @param pVCpu The cross context virtual CPU structure.
7894 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7895 * @thread EMT(pVCpu)
7896 */
7897VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7898{
7899 NOREF(fGlobalInit);
7900
7901 switch (enmEvent)
7902 {
7903 case RTTHREADCTXEVENT_OUT:
7904 {
7905 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7906 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7907 VMCPU_ASSERT_EMT(pVCpu);
7908
7909 /* No longjmps (logger flushes, locks) in this fragile context. */
7910 VMMRZCallRing3Disable(pVCpu);
7911 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7912
7913 /*
7914 * Restore host-state (FPU, debug etc.)
7915 */
7916 if (!pVCpu->hm.s.fLeaveDone)
7917 {
7918 /*
7919 * Do -not- import the guest-state here as we might already be in the middle of importing
7920 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7921 */
7922 hmR0VmxLeave(pVCpu, false /* fImportState */);
7923 pVCpu->hm.s.fLeaveDone = true;
7924 }
7925
7926 /* Leave HM context, takes care of local init (term). */
7927 int rc = HMR0LeaveCpu(pVCpu);
7928 AssertRC(rc); NOREF(rc);
7929
7930 /* Restore longjmp state. */
7931 VMMRZCallRing3Enable(pVCpu);
7932 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7933 break;
7934 }
7935
7936 case RTTHREADCTXEVENT_IN:
7937 {
7938 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7939 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7940 VMCPU_ASSERT_EMT(pVCpu);
7941
7942 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7943 VMMRZCallRing3Disable(pVCpu);
7944 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7945
7946 /* Initialize the bare minimum state required for HM. This takes care of
7947 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7948 int rc = hmR0EnterCpu(pVCpu);
7949 AssertRC(rc);
7950 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7951 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7952
7953 /* Load the active VMCS as the current one. */
7954 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7955 {
7956 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7957 AssertRC(rc); NOREF(rc);
7958 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7959 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7960 }
7961 pVCpu->hm.s.fLeaveDone = false;
7962
7963 /* Restore longjmp state. */
7964 VMMRZCallRing3Enable(pVCpu);
7965 break;
7966 }
7967
7968 default:
7969 break;
7970 }
7971}
7972
7973
7974/**
7975 * Exports the host state into the VMCS host-state area.
7976 * Sets up the VM-exit MSR-load area.
7977 *
7978 * The CPU state will be loaded from these fields on every successful VM-exit.
7979 *
7980 * @returns VBox status code.
7981 * @param pVCpu The cross context virtual CPU structure.
7982 *
7983 * @remarks No-long-jump zone!!!
7984 */
7985static int hmR0VmxExportHostState(PVMCPU pVCpu)
7986{
7987 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7988
7989 int rc = VINF_SUCCESS;
7990 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7991 {
7992 rc = hmR0VmxExportHostControlRegs();
7993 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7994
7995 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7996 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7997
7998 rc = hmR0VmxExportHostMsrs(pVCpu);
7999 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8000
8001 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
8002 }
8003 return rc;
8004}
8005
8006
8007/**
8008 * Saves the host state in the VMCS host-state.
8009 *
8010 * @returns VBox status code.
8011 * @param pVCpu The cross context virtual CPU structure.
8012 *
8013 * @remarks No-long-jump zone!!!
8014 */
8015VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
8016{
8017 AssertPtr(pVCpu);
8018 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8019
8020 /*
8021 * Export the host state here while entering HM context.
8022 * When thread-context hooks are used, we might get preempted and have to re-save the host
8023 * state but most of the time we won't be, so do it here before we disable interrupts.
8024 */
8025 return hmR0VmxExportHostState(pVCpu);
8026}
8027
8028
8029/**
8030 * Exports the guest state into the VMCS guest-state area.
8031 *
8032 * The will typically be done before VM-entry when the guest-CPU state and the
8033 * VMCS state may potentially be out of sync.
8034 *
8035 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
8036 * VM-entry controls.
8037 * Sets up the appropriate VMX non-root function to execute guest code based on
8038 * the guest CPU mode.
8039 *
8040 * @returns VBox strict status code.
8041 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8042 * without unrestricted guest access and the VMMDev is not presently
8043 * mapped (e.g. EFI32).
8044 *
8045 * @param pVCpu The cross context virtual CPU structure.
8046 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8047 * out-of-sync. Make sure to update the required fields
8048 * before using them.
8049 *
8050 * @remarks No-long-jump zone!!!
8051 */
8052static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8053{
8054 AssertPtr(pVCpu);
8055 AssertPtr(pMixedCtx);
8056 HMVMX_ASSERT_PREEMPT_SAFE();
8057
8058 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8059
8060 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8061
8062 /* Determine real-on-v86 mode. */
8063 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8064 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8065 && CPUMIsGuestInRealModeEx(pMixedCtx))
8066 {
8067 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8068 }
8069
8070 /*
8071 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8072 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8073 */
8074 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8075 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8076
8077 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8078 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8079 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8080
8081 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8082 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8083 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8084
8085 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8086 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8087
8088 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8089 if (rcStrict == VINF_SUCCESS)
8090 { /* likely */ }
8091 else
8092 {
8093 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8094 return rcStrict;
8095 }
8096
8097 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8098 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8099
8100 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8101 may alter controls if we determine we don't have to swap EFER after all. */
8102 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8103 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8104
8105 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8106 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8107
8108 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8109 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8110 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8111
8112 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8113 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8114 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8115 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8116 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8117 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8118
8119 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8120 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8121 | HM_CHANGED_GUEST_CR2
8122 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8123 | HM_CHANGED_GUEST_X87
8124 | HM_CHANGED_GUEST_SSE_AVX
8125 | HM_CHANGED_GUEST_OTHER_XSAVE
8126 | HM_CHANGED_GUEST_XCRx
8127 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8128 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8129 | HM_CHANGED_GUEST_TSC_AUX
8130 | HM_CHANGED_GUEST_OTHER_MSRS
8131 | HM_CHANGED_GUEST_HWVIRT
8132 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8133
8134 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8135 return rc;
8136}
8137
8138
8139/**
8140 * Exports the state shared between the host and guest into the VMCS.
8141 *
8142 * @param pVCpu The cross context virtual CPU structure.
8143 * @param pCtx Pointer to the guest-CPU context.
8144 *
8145 * @remarks No-long-jump zone!!!
8146 */
8147static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8148{
8149 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8150 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8151
8152 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8153 {
8154 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8155 AssertRC(rc);
8156 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8157
8158 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8159 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8160 {
8161 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8162 AssertRC(rc);
8163 }
8164 }
8165
8166 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8167 {
8168 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8169 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8170 }
8171
8172 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8173 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8174}
8175
8176
8177/**
8178 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8179 *
8180 * @returns Strict VBox status code (i.e. informational status codes too).
8181 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8182 * without unrestricted guest access and the VMMDev is not presently
8183 * mapped (e.g. EFI32).
8184 *
8185 * @param pVCpu The cross context virtual CPU structure.
8186 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8187 * out-of-sync. Make sure to update the required fields
8188 * before using them.
8189 *
8190 * @remarks No-long-jump zone!!!
8191 */
8192static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8193{
8194 HMVMX_ASSERT_PREEMPT_SAFE();
8195 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8196 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8197
8198#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8199 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8200#endif
8201
8202 /*
8203 * For many exits it's only RIP that changes and hence try to export it first
8204 * without going through a lot of change flag checks.
8205 */
8206 VBOXSTRICTRC rcStrict;
8207 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8208 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8209 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8210 {
8211 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8212 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8213 { /* likely */}
8214 else
8215 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8216 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8217 }
8218 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8219 {
8220 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8221 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8222 { /* likely */}
8223 else
8224 {
8225 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8226 VBOXSTRICTRC_VAL(rcStrict)));
8227 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8228 return rcStrict;
8229 }
8230 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8231 }
8232 else
8233 rcStrict = VINF_SUCCESS;
8234
8235#ifdef VBOX_STRICT
8236 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8237 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8238 RT_UNTRUSTED_NONVOLATILE_COPY_FENCE();
8239 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8240 ("fCtxChanged=%#RX64\n", fCtxChanged));
8241#endif
8242 return rcStrict;
8243}
8244
8245
8246/**
8247 * Does the preparations before executing guest code in VT-x.
8248 *
8249 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8250 * recompiler/IEM. We must be cautious what we do here regarding committing
8251 * guest-state information into the VMCS assuming we assuredly execute the
8252 * guest in VT-x mode.
8253 *
8254 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8255 * the common-state (TRPM/forceflags), we must undo those changes so that the
8256 * recompiler/IEM can (and should) use them when it resumes guest execution.
8257 * Otherwise such operations must be done when we can no longer exit to ring-3.
8258 *
8259 * @returns Strict VBox status code (i.e. informational status codes too).
8260 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8261 * have been disabled.
8262 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8263 * double-fault into the guest.
8264 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8265 * dispatched directly.
8266 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8267 *
8268 * @param pVCpu The cross context virtual CPU structure.
8269 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8270 * out-of-sync. Make sure to update the required fields
8271 * before using them.
8272 * @param pVmxTransient Pointer to the VMX transient structure.
8273 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8274 * us ignore some of the reasons for returning to
8275 * ring-3, and return VINF_EM_DBG_STEPPED if event
8276 * dispatching took place.
8277 */
8278static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8279{
8280 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8281
8282#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8283 PGMRZDynMapFlushAutoSet(pVCpu);
8284#endif
8285
8286 /* Check force flag actions that might require us to go back to ring-3. */
8287 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8288 if (rcStrict == VINF_SUCCESS)
8289 { /* FFs doesn't get set all the time. */ }
8290 else
8291 return rcStrict;
8292
8293 /*
8294 * Setup the virtualized-APIC accesses.
8295 *
8296 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8297 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8298 *
8299 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8300 */
8301 PVM pVM = pVCpu->CTX_SUFF(pVM);
8302 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8303 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8304 && PDMHasApic(pVM))
8305 {
8306 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8307 Assert(u64MsrApicBase);
8308 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8309
8310 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8311
8312 /* Unalias any existing mapping. */
8313 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8314 AssertRCReturn(rc, rc);
8315
8316 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8317 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8318 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8319 AssertRCReturn(rc, rc);
8320
8321 /* Update the per-VCPU cache of the APIC base MSR. */
8322 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8323 }
8324
8325 if (TRPMHasTrap(pVCpu))
8326 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8327 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8328
8329 /*
8330 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8331 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8332 * also result in triple-faulting the VM.
8333 */
8334 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8335 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8336 { /* likely */ }
8337 else
8338 {
8339 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8340 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8341 return rcStrict;
8342 }
8343
8344 /*
8345 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8346 * import CR3 themselves. We will need to update them here as even as late as the above
8347 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8348 * the below force flags to be set.
8349 */
8350 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8351 {
8352 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8353 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8354 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8355 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8356 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8357 }
8358 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8359 {
8360 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8361 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8362 }
8363
8364 /*
8365 * No longjmps to ring-3 from this point on!!!
8366 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8367 * This also disables flushing of the R0-logger instance (if any).
8368 */
8369 VMMRZCallRing3Disable(pVCpu);
8370
8371 /*
8372 * Export the guest state bits.
8373 *
8374 * We cannot perform longjmps while loading the guest state because we do not preserve the
8375 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8376 * CPU migration.
8377 *
8378 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8379 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8380 * Hence, loading of the guest state needs to be done -after- injection of events.
8381 */
8382 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8383 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8384 { /* likely */ }
8385 else
8386 {
8387 VMMRZCallRing3Enable(pVCpu);
8388 return rcStrict;
8389 }
8390
8391 /*
8392 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8393 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8394 * preemption disabled for a while. Since this is purly to aid the
8395 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8396 * disable interrupt on NT.
8397 *
8398 * We need to check for force-flags that could've possible been altered since we last
8399 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8400 * see @bugref{6398}).
8401 *
8402 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8403 * to ring-3 before executing guest code.
8404 */
8405 pVmxTransient->fEFlags = ASMIntDisableFlags();
8406
8407 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8408 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8409 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8410 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8411 {
8412 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8413 {
8414 pVCpu->hm.s.Event.fPending = false;
8415
8416 /*
8417 * We've injected any pending events. This is really the point of no return (to ring-3).
8418 *
8419 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8420 * returns from this function, so don't enable them here.
8421 */
8422 return VINF_SUCCESS;
8423 }
8424
8425 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8426 rcStrict = VINF_EM_RAW_INTERRUPT;
8427 }
8428 else
8429 {
8430 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8431 rcStrict = VINF_EM_RAW_TO_R3;
8432 }
8433
8434 ASMSetFlags(pVmxTransient->fEFlags);
8435 VMMRZCallRing3Enable(pVCpu);
8436
8437 return rcStrict;
8438}
8439
8440
8441/**
8442 * Prepares to run guest code in VT-x and we've committed to doing so. This
8443 * means there is no backing out to ring-3 or anywhere else at this
8444 * point.
8445 *
8446 * @param pVCpu The cross context virtual CPU structure.
8447 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8448 * out-of-sync. Make sure to update the required fields
8449 * before using them.
8450 * @param pVmxTransient Pointer to the VMX transient structure.
8451 *
8452 * @remarks Called with preemption disabled.
8453 * @remarks No-long-jump zone!!!
8454 */
8455static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8456{
8457 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8458 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8459 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8460
8461 /*
8462 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8463 */
8464 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8465 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8466
8467 PVM pVM = pVCpu->CTX_SUFF(pVM);
8468 if (!CPUMIsGuestFPUStateActive(pVCpu))
8469 {
8470 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8471 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8472 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8473 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8474 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8475 }
8476
8477 /*
8478 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8479 */
8480 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8481 && pVCpu->hm.s.vmx.cMsrs > 0)
8482 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8483
8484 /*
8485 * Re-save the host state bits as we may've been preempted (only happens when
8486 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8487 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8488 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8489 * See @bugref{8432}.
8490 */
8491 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8492 {
8493 int rc = hmR0VmxExportHostState(pVCpu);
8494 AssertRC(rc);
8495 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8496 }
8497 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8498
8499 /*
8500 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8501 */
8502 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8503 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8504 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8505
8506 /* Store status of the shared guest-host state at the time of VM-entry. */
8507#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8508 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8509 {
8510 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8511 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8512 }
8513 else
8514#endif
8515 {
8516 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8517 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8518 }
8519
8520 /*
8521 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8522 */
8523 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8524 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8525
8526 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8527 RTCPUID idCurrentCpu = pCpu->idCpu;
8528 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8529 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8530 {
8531 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8532 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8533 }
8534
8535 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8536 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8537 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8538 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8539
8540 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8541
8542 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8543 to start executing. */
8544
8545 /*
8546 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8547 */
8548 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8549 {
8550 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8551 {
8552 bool fMsrUpdated;
8553 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8554 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8555 &fMsrUpdated);
8556 AssertRC(rc2);
8557 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8558 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8559 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8560 }
8561 else
8562 {
8563 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8564 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8565 }
8566 }
8567
8568 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8569 {
8570 bool fMsrUpdated;
8571 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8572 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8573 &fMsrUpdated);
8574 AssertRC(rc2);
8575 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8576 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8577 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8578 }
8579
8580#ifdef VBOX_STRICT
8581 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8582 hmR0VmxCheckHostEferMsr(pVCpu);
8583 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8584#endif
8585#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8586 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
8587 {
8588 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8589 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8590 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8591 }
8592#endif
8593}
8594
8595
8596/**
8597 * Performs some essential restoration of state after running guest code in
8598 * VT-x.
8599 *
8600 * @param pVCpu The cross context virtual CPU structure.
8601 * @param pVmxTransient Pointer to the VMX transient structure.
8602 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8603 *
8604 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8605 *
8606 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8607 * unconditionally when it is safe to do so.
8608 */
8609static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8610{
8611 uint64_t const uHostTsc = ASMReadTSC();
8612 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8613
8614 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8615 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8616 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8617 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8618 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8619 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8620
8621 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8622 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8623
8624 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8625 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8626 Assert(!ASMIntAreEnabled());
8627 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8628
8629#if HC_ARCH_BITS == 64
8630 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8631#endif
8632#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8633 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8634 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8635 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8636#else
8637 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8638#endif
8639#ifdef VBOX_STRICT
8640 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8641#endif
8642 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8643
8644 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8645 uint32_t uExitReason;
8646 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8647 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8648 AssertRC(rc);
8649 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8650 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8651
8652 if (rcVMRun == VINF_SUCCESS)
8653 {
8654 /*
8655 * Update the VM-exit history array here even if the VM-entry failed due to:
8656 * - Invalid guest state.
8657 * - MSR loading.
8658 * - Machine-check event.
8659 *
8660 * In any of the above cases we will still have a "valid" VM-exit reason
8661 * despite @a fVMEntryFailed being false.
8662 *
8663 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8664 *
8665 * Note! We don't have CS or RIP at this point. Will probably address that later
8666 * by amending the history entry added here.
8667 */
8668 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8669 UINT64_MAX, uHostTsc);
8670
8671 if (!pVmxTransient->fVMEntryFailed)
8672 {
8673 VMMRZCallRing3Enable(pVCpu);
8674
8675 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8676 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8677
8678#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8679 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8680 AssertRC(rc);
8681#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8682 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8683 AssertRC(rc);
8684#else
8685 /*
8686 * Import the guest-interruptibility state always as we need it while evaluating
8687 * injecting events on re-entry.
8688 *
8689 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8690 * checking for real-mode while exporting the state because all bits that cause
8691 * mode changes wrt CR0 are intercepted.
8692 */
8693 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8694 AssertRC(rc);
8695#endif
8696
8697 /*
8698 * Sync the TPR shadow with our APIC state.
8699 */
8700 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8701 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8702 {
8703 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8704 AssertRC(rc);
8705 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8706 }
8707
8708 return;
8709 }
8710 }
8711 else
8712 {
8713 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8714 }
8715
8716 VMMRZCallRing3Enable(pVCpu);
8717}
8718
8719
8720/**
8721 * Runs the guest code using VT-x the normal way.
8722 *
8723 * @returns VBox status code.
8724 * @param pVCpu The cross context virtual CPU structure.
8725 * @param pCtx Pointer to the guest-CPU context.
8726 *
8727 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8728 */
8729static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8730{
8731 VMXTRANSIENT VmxTransient;
8732 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8733 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8734 uint32_t cLoops = 0;
8735
8736 for (;; cLoops++)
8737 {
8738 Assert(!HMR0SuspendPending());
8739 HMVMX_ASSERT_CPU_SAFE();
8740
8741 /* Preparatory work for running guest code, this may force us to return
8742 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8743 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8744 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8745 if (rcStrict != VINF_SUCCESS)
8746 break;
8747
8748 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8749 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8750 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8751
8752 /* Restore any residual host-state and save any bits shared between host
8753 and guest into the guest-CPU state. Re-enables interrupts! */
8754 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8755
8756 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8757 if (RT_SUCCESS(rcRun))
8758 { /* very likely */ }
8759 else
8760 {
8761 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8762 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8763 return rcRun;
8764 }
8765
8766 /* Profile the VM-exit. */
8767 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8768 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8769 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8770 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8771 HMVMX_START_EXIT_DISPATCH_PROF();
8772
8773 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8774
8775 /* Handle the VM-exit. */
8776#ifdef HMVMX_USE_FUNCTION_TABLE
8777 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8778#else
8779 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8780#endif
8781 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8782 if (rcStrict == VINF_SUCCESS)
8783 {
8784 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8785 continue; /* likely */
8786 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8787 rcStrict = VINF_EM_RAW_INTERRUPT;
8788 }
8789 break;
8790 }
8791
8792 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8793 return rcStrict;
8794}
8795
8796
8797
8798/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8799 * probes.
8800 *
8801 * The following few functions and associated structure contains the bloat
8802 * necessary for providing detailed debug events and dtrace probes as well as
8803 * reliable host side single stepping. This works on the principle of
8804 * "subclassing" the normal execution loop and workers. We replace the loop
8805 * method completely and override selected helpers to add necessary adjustments
8806 * to their core operation.
8807 *
8808 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8809 * any performance for debug and analysis features.
8810 *
8811 * @{
8812 */
8813
8814/**
8815 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8816 * the debug run loop.
8817 */
8818typedef struct VMXRUNDBGSTATE
8819{
8820 /** The RIP we started executing at. This is for detecting that we stepped. */
8821 uint64_t uRipStart;
8822 /** The CS we started executing with. */
8823 uint16_t uCsStart;
8824
8825 /** Whether we've actually modified the 1st execution control field. */
8826 bool fModifiedProcCtls : 1;
8827 /** Whether we've actually modified the 2nd execution control field. */
8828 bool fModifiedProcCtls2 : 1;
8829 /** Whether we've actually modified the exception bitmap. */
8830 bool fModifiedXcptBitmap : 1;
8831
8832 /** We desire the modified the CR0 mask to be cleared. */
8833 bool fClearCr0Mask : 1;
8834 /** We desire the modified the CR4 mask to be cleared. */
8835 bool fClearCr4Mask : 1;
8836 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8837 uint32_t fCpe1Extra;
8838 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8839 uint32_t fCpe1Unwanted;
8840 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8841 uint32_t fCpe2Extra;
8842 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8843 uint32_t bmXcptExtra;
8844 /** The sequence number of the Dtrace provider settings the state was
8845 * configured against. */
8846 uint32_t uDtraceSettingsSeqNo;
8847 /** VM-exits to check (one bit per VM-exit). */
8848 uint32_t bmExitsToCheck[3];
8849
8850 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8851 uint32_t fProcCtlsInitial;
8852 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8853 uint32_t fProcCtls2Initial;
8854 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8855 uint32_t bmXcptInitial;
8856} VMXRUNDBGSTATE;
8857AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8858typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8859
8860
8861/**
8862 * Initializes the VMXRUNDBGSTATE structure.
8863 *
8864 * @param pVCpu The cross context virtual CPU structure of the
8865 * calling EMT.
8866 * @param pCtx The CPU register context to go with @a pVCpu.
8867 * @param pDbgState The structure to initialize.
8868 */
8869static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8870{
8871 pDbgState->uRipStart = pCtx->rip;
8872 pDbgState->uCsStart = pCtx->cs.Sel;
8873
8874 pDbgState->fModifiedProcCtls = false;
8875 pDbgState->fModifiedProcCtls2 = false;
8876 pDbgState->fModifiedXcptBitmap = false;
8877 pDbgState->fClearCr0Mask = false;
8878 pDbgState->fClearCr4Mask = false;
8879 pDbgState->fCpe1Extra = 0;
8880 pDbgState->fCpe1Unwanted = 0;
8881 pDbgState->fCpe2Extra = 0;
8882 pDbgState->bmXcptExtra = 0;
8883 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8884 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8885 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8886}
8887
8888
8889/**
8890 * Updates the VMSC fields with changes requested by @a pDbgState.
8891 *
8892 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8893 * immediately before executing guest code, i.e. when interrupts are disabled.
8894 * We don't check status codes here as we cannot easily assert or return in the
8895 * latter case.
8896 *
8897 * @param pVCpu The cross context virtual CPU structure.
8898 * @param pDbgState The debug state.
8899 */
8900static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8901{
8902 /*
8903 * Ensure desired flags in VMCS control fields are set.
8904 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8905 *
8906 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8907 * there should be no stale data in pCtx at this point.
8908 */
8909 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8910 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8911 {
8912 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8913 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8914 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8915 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8916 pDbgState->fModifiedProcCtls = true;
8917 }
8918
8919 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8920 {
8921 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8922 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8923 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8924 pDbgState->fModifiedProcCtls2 = true;
8925 }
8926
8927 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8928 {
8929 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8930 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8931 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8932 pDbgState->fModifiedXcptBitmap = true;
8933 }
8934
8935 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32Cr0Mask != 0)
8936 {
8937 pVCpu->hm.s.vmx.u32Cr0Mask = 0;
8938 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8939 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8940 }
8941
8942 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32Cr4Mask != 0)
8943 {
8944 pVCpu->hm.s.vmx.u32Cr4Mask = 0;
8945 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8946 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8947 }
8948}
8949
8950
8951static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8952{
8953 /*
8954 * Restore VM-exit control settings as we may not reenter this function the
8955 * next time around.
8956 */
8957 /* We reload the initial value, trigger what we can of recalculations the
8958 next time around. From the looks of things, that's all that's required atm. */
8959 if (pDbgState->fModifiedProcCtls)
8960 {
8961 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8962 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8963 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8964 AssertRCReturn(rc2, rc2);
8965 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8966 }
8967
8968 /* We're currently the only ones messing with this one, so just restore the
8969 cached value and reload the field. */
8970 if ( pDbgState->fModifiedProcCtls2
8971 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8972 {
8973 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8974 AssertRCReturn(rc2, rc2);
8975 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8976 }
8977
8978 /* If we've modified the exception bitmap, we restore it and trigger
8979 reloading and partial recalculation the next time around. */
8980 if (pDbgState->fModifiedXcptBitmap)
8981 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8982
8983 return rcStrict;
8984}
8985
8986
8987/**
8988 * Configures VM-exit controls for current DBGF and DTrace settings.
8989 *
8990 * This updates @a pDbgState and the VMCS execution control fields to reflect
8991 * the necessary VM-exits demanded by DBGF and DTrace.
8992 *
8993 * @param pVCpu The cross context virtual CPU structure.
8994 * @param pDbgState The debug state.
8995 * @param pVmxTransient Pointer to the VMX transient structure. May update
8996 * fUpdateTscOffsettingAndPreemptTimer.
8997 */
8998static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8999{
9000 /*
9001 * Take down the dtrace serial number so we can spot changes.
9002 */
9003 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
9004 ASMCompilerBarrier();
9005
9006 /*
9007 * We'll rebuild most of the middle block of data members (holding the
9008 * current settings) as we go along here, so start by clearing it all.
9009 */
9010 pDbgState->bmXcptExtra = 0;
9011 pDbgState->fCpe1Extra = 0;
9012 pDbgState->fCpe1Unwanted = 0;
9013 pDbgState->fCpe2Extra = 0;
9014 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
9015 pDbgState->bmExitsToCheck[i] = 0;
9016
9017 /*
9018 * Software interrupts (INT XXh) - no idea how to trigger these...
9019 */
9020 PVM pVM = pVCpu->CTX_SUFF(pVM);
9021 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
9022 || VBOXVMM_INT_SOFTWARE_ENABLED())
9023 {
9024 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9025 }
9026
9027 /*
9028 * INT3 breakpoints - triggered by #BP exceptions.
9029 */
9030 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
9031 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9032
9033 /*
9034 * Exception bitmap and XCPT events+probes.
9035 */
9036 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
9037 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
9038 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
9039
9040 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
9041 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9042 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9043 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9044 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9045 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9046 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9047 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9048 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9049 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9050 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9051 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9052 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9053 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9054 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9055 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9056 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9057 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9058
9059 if (pDbgState->bmXcptExtra)
9060 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9061
9062 /*
9063 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9064 *
9065 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9066 * So, when adding/changing/removing please don't forget to update it.
9067 *
9068 * Some of the macros are picking up local variables to save horizontal space,
9069 * (being able to see it in a table is the lesser evil here).
9070 */
9071#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9072 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9073 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9074#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9075 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9076 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9077 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9078 } else do { } while (0)
9079#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9080 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9081 { \
9082 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9083 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9084 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9085 } else do { } while (0)
9086#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9087 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9088 { \
9089 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9090 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9091 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9092 } else do { } while (0)
9093#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9094 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9095 { \
9096 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9097 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9098 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9099 } else do { } while (0)
9100
9101 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9102 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9103 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9104 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9105 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9106
9107 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9108 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9109 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9110 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9111 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9112 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9113 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9114 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9115 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9116 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9117 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9118 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9119 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9120 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9121 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9122 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9123 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9124 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9125 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9126 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9127 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9128 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9129 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9130 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9131 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9132 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9133 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9134 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9135 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9137 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9138 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9139 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9140 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9141 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9142 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9143
9144 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9145 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9146 {
9147 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9148 | CPUMCTX_EXTRN_CR4
9149 | CPUMCTX_EXTRN_APIC_TPR);
9150 AssertRC(rc);
9151
9152#if 0 /** @todo fix me */
9153 pDbgState->fClearCr0Mask = true;
9154 pDbgState->fClearCr4Mask = true;
9155#endif
9156 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9157 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9158 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9159 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9160 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9161 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9162 require clearing here and in the loop if we start using it. */
9163 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9164 }
9165 else
9166 {
9167 if (pDbgState->fClearCr0Mask)
9168 {
9169 pDbgState->fClearCr0Mask = false;
9170 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9171 }
9172 if (pDbgState->fClearCr4Mask)
9173 {
9174 pDbgState->fClearCr4Mask = false;
9175 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9176 }
9177 }
9178 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9179 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9180
9181 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9182 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9183 {
9184 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9185 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9186 }
9187 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9188 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9189
9190 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9191 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9192 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9193 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9194 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9195 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9196 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9197 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9198#if 0 /** @todo too slow, fix handler. */
9199 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9200#endif
9201 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9202
9203 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9204 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9205 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9206 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9207 {
9208 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9209 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9210 }
9211 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9212 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9213 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9214 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9215
9216 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9217 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9218 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9219 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9220 {
9221 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9222 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9223 }
9224 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9225 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9226 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9227 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9228
9229 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9230 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9231 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9232 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9233 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9234 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9235 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9236 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9237 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9238 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9239 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9240 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9241 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9242 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9243 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9244 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9245 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9246 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9247 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9248 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9249 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9250 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9251
9252#undef IS_EITHER_ENABLED
9253#undef SET_ONLY_XBM_IF_EITHER_EN
9254#undef SET_CPE1_XBM_IF_EITHER_EN
9255#undef SET_CPEU_XBM_IF_EITHER_EN
9256#undef SET_CPE2_XBM_IF_EITHER_EN
9257
9258 /*
9259 * Sanitize the control stuff.
9260 */
9261 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9262 if (pDbgState->fCpe2Extra)
9263 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9264 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9265 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9266 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9267 {
9268 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9269 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9270 }
9271
9272 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9273 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9274 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9275 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9276}
9277
9278
9279/**
9280 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9281 * appropriate.
9282 *
9283 * The caller has checked the VM-exit against the
9284 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9285 * already, so we don't have to do that either.
9286 *
9287 * @returns Strict VBox status code (i.e. informational status codes too).
9288 * @param pVCpu The cross context virtual CPU structure.
9289 * @param pMixedCtx Pointer to the guest-CPU context.
9290 * @param pVmxTransient Pointer to the VMX-transient structure.
9291 * @param uExitReason The VM-exit reason.
9292 *
9293 * @remarks The name of this function is displayed by dtrace, so keep it short
9294 * and to the point. No longer than 33 chars long, please.
9295 */
9296static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9297 uint32_t uExitReason)
9298{
9299 /*
9300 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9301 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9302 *
9303 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9304 * does. Must add/change/remove both places. Same ordering, please.
9305 *
9306 * Added/removed events must also be reflected in the next section
9307 * where we dispatch dtrace events.
9308 */
9309 bool fDtrace1 = false;
9310 bool fDtrace2 = false;
9311 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9312 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9313 uint32_t uEventArg = 0;
9314#define SET_EXIT(a_EventSubName) \
9315 do { \
9316 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9317 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9318 } while (0)
9319#define SET_BOTH(a_EventSubName) \
9320 do { \
9321 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9322 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9323 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9324 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9325 } while (0)
9326 switch (uExitReason)
9327 {
9328 case VMX_EXIT_MTF:
9329 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9330
9331 case VMX_EXIT_XCPT_OR_NMI:
9332 {
9333 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9334 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9335 {
9336 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9337 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9338 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9339 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9340 {
9341 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9342 {
9343 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9344 uEventArg = pVmxTransient->uExitIntErrorCode;
9345 }
9346 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9347 switch (enmEvent1)
9348 {
9349 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9350 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9351 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9352 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9353 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9354 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9355 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9356 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9357 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9358 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9359 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9360 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9361 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9362 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9363 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9364 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9365 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9366 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9367 default: break;
9368 }
9369 }
9370 else
9371 AssertFailed();
9372 break;
9373
9374 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9375 uEventArg = idxVector;
9376 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9377 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9378 break;
9379 }
9380 break;
9381 }
9382
9383 case VMX_EXIT_TRIPLE_FAULT:
9384 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9385 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9386 break;
9387 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9388 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9389 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9390 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9391 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9392
9393 /* Instruction specific VM-exits: */
9394 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9395 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9396 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9397 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9398 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9399 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9400 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9401 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9402 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9403 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9404 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9405 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9406 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9407 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9408 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9409 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9410 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9411 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9412 case VMX_EXIT_MOV_CRX:
9413 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9414 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9415 SET_BOTH(CRX_READ);
9416 else
9417 SET_BOTH(CRX_WRITE);
9418 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9419 break;
9420 case VMX_EXIT_MOV_DRX:
9421 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9422 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9423 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9424 SET_BOTH(DRX_READ);
9425 else
9426 SET_BOTH(DRX_WRITE);
9427 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9428 break;
9429 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9430 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9431 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9432 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9433 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9434 case VMX_EXIT_XDTR_ACCESS:
9435 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9436 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9437 {
9438 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9439 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9440 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9441 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9442 }
9443 break;
9444
9445 case VMX_EXIT_TR_ACCESS:
9446 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9447 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9448 {
9449 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9450 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9451 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9452 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9453 }
9454 break;
9455
9456 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9457 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9458 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9459 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9460 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9461 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9462 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9463 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9464 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9465 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9466 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9467
9468 /* Events that aren't relevant at this point. */
9469 case VMX_EXIT_EXT_INT:
9470 case VMX_EXIT_INT_WINDOW:
9471 case VMX_EXIT_NMI_WINDOW:
9472 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9473 case VMX_EXIT_PREEMPT_TIMER:
9474 case VMX_EXIT_IO_INSTR:
9475 break;
9476
9477 /* Errors and unexpected events. */
9478 case VMX_EXIT_INIT_SIGNAL:
9479 case VMX_EXIT_SIPI:
9480 case VMX_EXIT_IO_SMI:
9481 case VMX_EXIT_SMI:
9482 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9483 case VMX_EXIT_ERR_MSR_LOAD:
9484 case VMX_EXIT_ERR_MACHINE_CHECK:
9485 break;
9486
9487 default:
9488 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9489 break;
9490 }
9491#undef SET_BOTH
9492#undef SET_EXIT
9493
9494 /*
9495 * Dtrace tracepoints go first. We do them here at once so we don't
9496 * have to copy the guest state saving and stuff a few dozen times.
9497 * Down side is that we've got to repeat the switch, though this time
9498 * we use enmEvent since the probes are a subset of what DBGF does.
9499 */
9500 if (fDtrace1 || fDtrace2)
9501 {
9502 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9503 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9504 switch (enmEvent1)
9505 {
9506 /** @todo consider which extra parameters would be helpful for each probe. */
9507 case DBGFEVENT_END: break;
9508 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9510 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9517 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9518 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9519 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9520 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9521 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9526 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9527 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9528 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9532 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9533 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9534 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9535 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9536 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9537 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9538 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9539 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9540 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9541 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9542 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9548 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9549 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9550 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9551 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9552 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9553 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9554 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9574 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9575 }
9576 switch (enmEvent2)
9577 {
9578 /** @todo consider which extra parameters would be helpful for each probe. */
9579 case DBGFEVENT_END: break;
9580 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9582 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9590 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9591 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9592 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9593 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9594 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9595 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9596 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9597 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9598 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9599 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9600 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9601 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9602 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9603 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9604 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9605 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9606 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9607 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9608 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9609 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9610 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9611 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9612 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9613 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9614 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9615 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9616 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9617 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9618 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9619 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9620 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9621 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9622 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9623 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9624 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9625 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9626 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9627 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9628 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9629 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9630 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9631 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9632 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9633 }
9634 }
9635
9636 /*
9637 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9638 * the DBGF call will do a full check).
9639 *
9640 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9641 * Note! If we have to events, we prioritize the first, i.e. the instruction
9642 * one, in order to avoid event nesting.
9643 */
9644 PVM pVM = pVCpu->CTX_SUFF(pVM);
9645 if ( enmEvent1 != DBGFEVENT_END
9646 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9647 {
9648 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9649 if (rcStrict != VINF_SUCCESS)
9650 return rcStrict;
9651 }
9652 else if ( enmEvent2 != DBGFEVENT_END
9653 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9654 {
9655 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9656 if (rcStrict != VINF_SUCCESS)
9657 return rcStrict;
9658 }
9659
9660 return VINF_SUCCESS;
9661}
9662
9663
9664/**
9665 * Single-stepping VM-exit filtering.
9666 *
9667 * This is preprocessing the VM-exits and deciding whether we've gotten far
9668 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9669 * handling is performed.
9670 *
9671 * @returns Strict VBox status code (i.e. informational status codes too).
9672 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9673 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9674 * out-of-sync. Make sure to update the required
9675 * fields before using them.
9676 * @param pVmxTransient Pointer to the VMX-transient structure.
9677 * @param uExitReason The VM-exit reason.
9678 * @param pDbgState The debug state.
9679 */
9680DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9681 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9682{
9683 /*
9684 * Expensive (saves context) generic dtrace VM-exit probe.
9685 */
9686 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9687 { /* more likely */ }
9688 else
9689 {
9690 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9691 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9692 AssertRC(rc);
9693 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9694 }
9695
9696 /*
9697 * Check for host NMI, just to get that out of the way.
9698 */
9699 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9700 { /* normally likely */ }
9701 else
9702 {
9703 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9704 AssertRCReturn(rc2, rc2);
9705 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9706 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9707 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9708 }
9709
9710 /*
9711 * Check for single stepping event if we're stepping.
9712 */
9713 if (pVCpu->hm.s.fSingleInstruction)
9714 {
9715 switch (uExitReason)
9716 {
9717 case VMX_EXIT_MTF:
9718 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9719
9720 /* Various events: */
9721 case VMX_EXIT_XCPT_OR_NMI:
9722 case VMX_EXIT_EXT_INT:
9723 case VMX_EXIT_TRIPLE_FAULT:
9724 case VMX_EXIT_INT_WINDOW:
9725 case VMX_EXIT_NMI_WINDOW:
9726 case VMX_EXIT_TASK_SWITCH:
9727 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9728 case VMX_EXIT_APIC_ACCESS:
9729 case VMX_EXIT_EPT_VIOLATION:
9730 case VMX_EXIT_EPT_MISCONFIG:
9731 case VMX_EXIT_PREEMPT_TIMER:
9732
9733 /* Instruction specific VM-exits: */
9734 case VMX_EXIT_CPUID:
9735 case VMX_EXIT_GETSEC:
9736 case VMX_EXIT_HLT:
9737 case VMX_EXIT_INVD:
9738 case VMX_EXIT_INVLPG:
9739 case VMX_EXIT_RDPMC:
9740 case VMX_EXIT_RDTSC:
9741 case VMX_EXIT_RSM:
9742 case VMX_EXIT_VMCALL:
9743 case VMX_EXIT_VMCLEAR:
9744 case VMX_EXIT_VMLAUNCH:
9745 case VMX_EXIT_VMPTRLD:
9746 case VMX_EXIT_VMPTRST:
9747 case VMX_EXIT_VMREAD:
9748 case VMX_EXIT_VMRESUME:
9749 case VMX_EXIT_VMWRITE:
9750 case VMX_EXIT_VMXOFF:
9751 case VMX_EXIT_VMXON:
9752 case VMX_EXIT_MOV_CRX:
9753 case VMX_EXIT_MOV_DRX:
9754 case VMX_EXIT_IO_INSTR:
9755 case VMX_EXIT_RDMSR:
9756 case VMX_EXIT_WRMSR:
9757 case VMX_EXIT_MWAIT:
9758 case VMX_EXIT_MONITOR:
9759 case VMX_EXIT_PAUSE:
9760 case VMX_EXIT_XDTR_ACCESS:
9761 case VMX_EXIT_TR_ACCESS:
9762 case VMX_EXIT_INVEPT:
9763 case VMX_EXIT_RDTSCP:
9764 case VMX_EXIT_INVVPID:
9765 case VMX_EXIT_WBINVD:
9766 case VMX_EXIT_XSETBV:
9767 case VMX_EXIT_RDRAND:
9768 case VMX_EXIT_INVPCID:
9769 case VMX_EXIT_VMFUNC:
9770 case VMX_EXIT_RDSEED:
9771 case VMX_EXIT_XSAVES:
9772 case VMX_EXIT_XRSTORS:
9773 {
9774 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9775 | CPUMCTX_EXTRN_CS);
9776 AssertRCReturn(rc, rc);
9777 if ( pMixedCtx->rip != pDbgState->uRipStart
9778 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9779 return VINF_EM_DBG_STEPPED;
9780 break;
9781 }
9782
9783 /* Errors and unexpected events: */
9784 case VMX_EXIT_INIT_SIGNAL:
9785 case VMX_EXIT_SIPI:
9786 case VMX_EXIT_IO_SMI:
9787 case VMX_EXIT_SMI:
9788 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9789 case VMX_EXIT_ERR_MSR_LOAD:
9790 case VMX_EXIT_ERR_MACHINE_CHECK:
9791 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9792 break;
9793
9794 default:
9795 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9796 break;
9797 }
9798 }
9799
9800 /*
9801 * Check for debugger event breakpoints and dtrace probes.
9802 */
9803 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9804 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9805 {
9806 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9807 if (rcStrict != VINF_SUCCESS)
9808 return rcStrict;
9809 }
9810
9811 /*
9812 * Normal processing.
9813 */
9814#ifdef HMVMX_USE_FUNCTION_TABLE
9815 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9816#else
9817 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9818#endif
9819}
9820
9821
9822/**
9823 * Single steps guest code using VT-x.
9824 *
9825 * @returns Strict VBox status code (i.e. informational status codes too).
9826 * @param pVCpu The cross context virtual CPU structure.
9827 * @param pCtx Pointer to the guest-CPU context.
9828 *
9829 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9830 */
9831static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9832{
9833 VMXTRANSIENT VmxTransient;
9834 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9835
9836 /* Set HMCPU indicators. */
9837 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9838 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9839 pVCpu->hm.s.fDebugWantRdTscExit = false;
9840 pVCpu->hm.s.fUsingDebugLoop = true;
9841
9842 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9843 VMXRUNDBGSTATE DbgState;
9844 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9845 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9846
9847 /*
9848 * The loop.
9849 */
9850 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9851 for (uint32_t cLoops = 0; ; cLoops++)
9852 {
9853 Assert(!HMR0SuspendPending());
9854 HMVMX_ASSERT_CPU_SAFE();
9855 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9856
9857 /*
9858 * Preparatory work for running guest code, this may force us to return
9859 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9860 */
9861 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9862 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9863 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9864 if (rcStrict != VINF_SUCCESS)
9865 break;
9866
9867 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9868 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9869
9870 /*
9871 * Now we can run the guest code.
9872 */
9873 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9874
9875 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9876
9877 /*
9878 * Restore any residual host-state and save any bits shared between host
9879 * and guest into the guest-CPU state. Re-enables interrupts!
9880 */
9881 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9882
9883 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9884 if (RT_SUCCESS(rcRun))
9885 { /* very likely */ }
9886 else
9887 {
9888 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9889 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9890 return rcRun;
9891 }
9892
9893 /* Profile the VM-exit. */
9894 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9895 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9896 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9897 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9898 HMVMX_START_EXIT_DISPATCH_PROF();
9899
9900 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9901
9902 /*
9903 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9904 */
9905 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9906 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9907 if (rcStrict != VINF_SUCCESS)
9908 break;
9909 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9910 {
9911 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9912 rcStrict = VINF_EM_RAW_INTERRUPT;
9913 break;
9914 }
9915
9916 /*
9917 * Stepping: Did the RIP change, if so, consider it a single step.
9918 * Otherwise, make sure one of the TFs gets set.
9919 */
9920 if (fStepping)
9921 {
9922 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9923 | CPUMCTX_EXTRN_CS);
9924 AssertRC(rc);
9925 if ( pCtx->rip != DbgState.uRipStart
9926 || pCtx->cs.Sel != DbgState.uCsStart)
9927 {
9928 rcStrict = VINF_EM_DBG_STEPPED;
9929 break;
9930 }
9931 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9932 }
9933
9934 /*
9935 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9936 */
9937 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9938 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9939 }
9940
9941 /*
9942 * Clear the X86_EFL_TF if necessary.
9943 */
9944 if (pVCpu->hm.s.fClearTrapFlag)
9945 {
9946 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9947 AssertRC(rc);
9948 pVCpu->hm.s.fClearTrapFlag = false;
9949 pCtx->eflags.Bits.u1TF = 0;
9950 }
9951 /** @todo there seems to be issues with the resume flag when the monitor trap
9952 * flag is pending without being used. Seen early in bios init when
9953 * accessing APIC page in protected mode. */
9954
9955 /*
9956 * Restore VM-exit control settings as we may not reenter this function the
9957 * next time around.
9958 */
9959 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9960
9961 /* Restore HMCPU indicators. */
9962 pVCpu->hm.s.fUsingDebugLoop = false;
9963 pVCpu->hm.s.fDebugWantRdTscExit = false;
9964 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9965
9966 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9967 return rcStrict;
9968}
9969
9970
9971/** @} */
9972
9973
9974/**
9975 * Checks if any expensive dtrace probes are enabled and we should go to the
9976 * debug loop.
9977 *
9978 * @returns true if we should use debug loop, false if not.
9979 */
9980static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9981{
9982 /* It's probably faster to OR the raw 32-bit counter variables together.
9983 Since the variables are in an array and the probes are next to one
9984 another (more or less), we have good locality. So, better read
9985 eight-nine cache lines ever time and only have one conditional, than
9986 128+ conditionals, right? */
9987 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9988 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9989 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9990 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9991 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9992 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9993 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9994 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9995 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9996 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9997 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9998 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9999 | VBOXVMM_XCPT_GP_ENABLED_RAW()
10000 | VBOXVMM_XCPT_PF_ENABLED_RAW()
10001 | VBOXVMM_XCPT_MF_ENABLED_RAW()
10002 | VBOXVMM_XCPT_AC_ENABLED_RAW()
10003 | VBOXVMM_XCPT_XF_ENABLED_RAW()
10004 | VBOXVMM_XCPT_VE_ENABLED_RAW()
10005 | VBOXVMM_XCPT_SX_ENABLED_RAW()
10006 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
10007 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
10008 ) != 0
10009 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
10010 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
10011 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
10012 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
10013 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
10014 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
10015 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
10016 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
10017 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
10018 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
10019 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
10020 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
10021 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
10022 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
10023 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
10024 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
10025 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
10026 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
10027 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
10028 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
10029 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
10030 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
10031 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
10032 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
10033 | VBOXVMM_INSTR_STR_ENABLED_RAW()
10034 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
10035 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
10036 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
10037 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
10038 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
10039 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
10040 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
10041 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10042 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10043 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10044 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10045 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10046 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10047 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10048 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10049 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10050 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10051 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10052 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10053 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10054 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10055 ) != 0
10056 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10057 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10058 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10059 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10060 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10061 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10062 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10063 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10064 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10065 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10066 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10067 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10068 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10069 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10070 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10071 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10072 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10073 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10074 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10075 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10076 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10077 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10078 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10079 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10080 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10081 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10082 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10083 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10084 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10085 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10086 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10087 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10088 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10089 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10090 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10091 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10092 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10093 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10094 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10095 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10096 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10097 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10098 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10099 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10100 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10101 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10102 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10103 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10104 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10105 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10106 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10107 ) != 0;
10108}
10109
10110
10111/**
10112 * Runs the guest code using VT-x.
10113 *
10114 * @returns Strict VBox status code (i.e. informational status codes too).
10115 * @param pVCpu The cross context virtual CPU structure.
10116 * @param pCtx Pointer to the guest-CPU context.
10117 */
10118VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10119{
10120 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10121 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10122 HMVMX_ASSERT_PREEMPT_SAFE();
10123
10124 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10125
10126 VBOXSTRICTRC rcStrict;
10127 if ( !pVCpu->hm.s.fUseDebugLoop
10128 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10129 && !DBGFIsStepping(pVCpu)
10130 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10131 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10132 else
10133 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10134
10135 if (rcStrict == VERR_EM_INTERPRETER)
10136 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10137 else if (rcStrict == VINF_EM_RESET)
10138 rcStrict = VINF_EM_TRIPLE_FAULT;
10139
10140 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10141 if (RT_FAILURE(rc2))
10142 {
10143 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10144 rcStrict = rc2;
10145 }
10146 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10147 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10148 return rcStrict;
10149}
10150
10151
10152#ifndef HMVMX_USE_FUNCTION_TABLE
10153DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10154{
10155#ifdef DEBUG_ramshankar
10156#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10157 do { \
10158 if (a_fSave != 0) \
10159 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10160 VBOXSTRICTRC rcStrict = a_CallExpr; \
10161 if (a_fSave != 0) \
10162 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10163 return rcStrict; \
10164 } while (0)
10165#else
10166# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10167#endif
10168 switch (rcReason)
10169 {
10170 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10171 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10172 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10173 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10174 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10175 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10176 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10177 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10178 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10179 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10180 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10181 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10182 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10183 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10184 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10185 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10186 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10187 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10188 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10189 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10190 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10191 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10192 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10193 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10194 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10195 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10196 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10197 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10198 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10199 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10200 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10201 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10202 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10203 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10204
10205 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10206 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10207 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10208 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10209 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10210 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10211 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10212 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10213 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10214
10215 case VMX_EXIT_VMCLEAR:
10216 case VMX_EXIT_VMLAUNCH:
10217 case VMX_EXIT_VMPTRLD:
10218 case VMX_EXIT_VMPTRST:
10219 case VMX_EXIT_VMREAD:
10220 case VMX_EXIT_VMRESUME:
10221 case VMX_EXIT_VMWRITE:
10222 case VMX_EXIT_VMXOFF:
10223 case VMX_EXIT_VMXON:
10224 case VMX_EXIT_INVEPT:
10225 case VMX_EXIT_INVVPID:
10226 case VMX_EXIT_VMFUNC:
10227 case VMX_EXIT_XSAVES:
10228 case VMX_EXIT_XRSTORS:
10229 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10230
10231 case VMX_EXIT_ENCLS:
10232 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10233 case VMX_EXIT_PML_FULL:
10234 default:
10235 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10236 }
10237#undef VMEXIT_CALL_RET
10238}
10239#endif /* !HMVMX_USE_FUNCTION_TABLE */
10240
10241
10242#ifdef VBOX_STRICT
10243/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10244# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10245 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10246
10247# define HMVMX_ASSERT_PREEMPT_CPUID() \
10248 do { \
10249 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10250 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10251 } while (0)
10252
10253# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10254 do { \
10255 AssertPtr(pVCpu); \
10256 AssertPtr(pMixedCtx); \
10257 AssertPtr(pVmxTransient); \
10258 Assert(pVmxTransient->fVMEntryFailed == false); \
10259 Assert(ASMIntAreEnabled()); \
10260 HMVMX_ASSERT_PREEMPT_SAFE(); \
10261 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10262 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10263 HMVMX_ASSERT_PREEMPT_SAFE(); \
10264 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10265 HMVMX_ASSERT_PREEMPT_CPUID(); \
10266 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10267 } while (0)
10268
10269# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10270 do { \
10271 Log4Func(("\n")); \
10272 } while (0)
10273#else /* nonstrict builds: */
10274# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10275 do { \
10276 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10277 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10278 } while (0)
10279# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10280#endif
10281
10282
10283/**
10284 * Advances the guest RIP by the specified number of bytes.
10285 *
10286 * @param pVCpu The cross context virtual CPU structure.
10287 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10288 * out-of-sync. Make sure to update the required fields
10289 * before using them.
10290 * @param cbInstr Number of bytes to advance the RIP by.
10291 *
10292 * @remarks No-long-jump zone!!!
10293 */
10294DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10295{
10296 /* Advance the RIP. */
10297 pMixedCtx->rip += cbInstr;
10298 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10299
10300 /* Update interrupt inhibition. */
10301 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10302 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10303 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10304}
10305
10306
10307/**
10308 * Advances the guest RIP after reading it from the VMCS.
10309 *
10310 * @returns VBox status code, no informational status codes.
10311 * @param pVCpu The cross context virtual CPU structure.
10312 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10313 * out-of-sync. Make sure to update the required fields
10314 * before using them.
10315 * @param pVmxTransient Pointer to the VMX transient structure.
10316 *
10317 * @remarks No-long-jump zone!!!
10318 */
10319static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10320{
10321 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10322 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10323 | CPUMCTX_EXTRN_RFLAGS);
10324 AssertRCReturn(rc, rc);
10325
10326 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10327
10328 /*
10329 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10330 * pending debug exception field as it takes care of priority of events.
10331 *
10332 * See Intel spec. 32.2.1 "Debug Exceptions".
10333 */
10334 if ( !pVCpu->hm.s.fSingleInstruction
10335 && pMixedCtx->eflags.Bits.u1TF)
10336 {
10337 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10338 AssertRCReturn(rc, rc);
10339 }
10340
10341 return VINF_SUCCESS;
10342}
10343
10344
10345/**
10346 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10347 * and update error record fields accordingly.
10348 *
10349 * @return VMX_IGS_* return codes.
10350 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10351 * wrong with the guest state.
10352 *
10353 * @param pVCpu The cross context virtual CPU structure.
10354 * @param pCtx Pointer to the guest-CPU state.
10355 *
10356 * @remarks This function assumes our cache of the VMCS controls
10357 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10358 */
10359static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10360{
10361#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10362#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10363 uError = (err); \
10364 break; \
10365 } else do { } while (0)
10366
10367 int rc;
10368 PVM pVM = pVCpu->CTX_SUFF(pVM);
10369 uint32_t uError = VMX_IGS_ERROR;
10370 uint32_t u32Val;
10371 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10372
10373 do
10374 {
10375 /*
10376 * CR0.
10377 */
10378 uint32_t fSetCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10379 uint32_t const fZapCr0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10380 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10381 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10382 if (fUnrestrictedGuest)
10383 fSetCr0 &= ~(X86_CR0_PE | X86_CR0_PG);
10384
10385 uint32_t u32GuestCr0;
10386 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32GuestCr0);
10387 AssertRCBreak(rc);
10388 HMVMX_CHECK_BREAK((u32GuestCr0 & fSetCr0) == fSetCr0, VMX_IGS_CR0_FIXED1);
10389 HMVMX_CHECK_BREAK(!(u32GuestCr0 & ~fZapCr0), VMX_IGS_CR0_FIXED0);
10390 if ( !fUnrestrictedGuest
10391 && (u32GuestCr0 & X86_CR0_PG)
10392 && !(u32GuestCr0 & X86_CR0_PE))
10393 {
10394 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10395 }
10396
10397 /*
10398 * CR4.
10399 */
10400 uint64_t const fSetCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10401 uint64_t const fZapCr4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10402
10403 uint32_t u32GuestCr4;
10404 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32GuestCr4);
10405 AssertRCBreak(rc);
10406 HMVMX_CHECK_BREAK((u32GuestCr4 & fSetCr4) == fSetCr4, VMX_IGS_CR4_FIXED1);
10407 HMVMX_CHECK_BREAK(!(u32GuestCr4 & ~fZapCr4), VMX_IGS_CR4_FIXED0);
10408
10409 /*
10410 * IA32_DEBUGCTL MSR.
10411 */
10412 uint64_t u64Val;
10413 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10414 AssertRCBreak(rc);
10415 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10416 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10417 {
10418 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10419 }
10420 uint64_t u64DebugCtlMsr = u64Val;
10421
10422#ifdef VBOX_STRICT
10423 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10424 AssertRCBreak(rc);
10425 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10426#endif
10427 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10428
10429 /*
10430 * RIP and RFLAGS.
10431 */
10432 uint32_t u32Eflags;
10433#if HC_ARCH_BITS == 64
10434 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10435 AssertRCBreak(rc);
10436 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10437 if ( !fLongModeGuest
10438 || !pCtx->cs.Attr.n.u1Long)
10439 {
10440 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10441 }
10442 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10443 * must be identical if the "IA-32e mode guest" VM-entry
10444 * control is 1 and CS.L is 1. No check applies if the
10445 * CPU supports 64 linear-address bits. */
10446
10447 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10448 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10449 AssertRCBreak(rc);
10450 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10451 VMX_IGS_RFLAGS_RESERVED);
10452 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10453 u32Eflags = u64Val;
10454#else
10455 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10456 AssertRCBreak(rc);
10457 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10458 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10459#endif
10460
10461 if ( fLongModeGuest
10462 || ( fUnrestrictedGuest
10463 && !(u32GuestCr0 & X86_CR0_PE)))
10464 {
10465 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10466 }
10467
10468 uint32_t u32EntryInfo;
10469 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10470 AssertRCBreak(rc);
10471 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10472 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10473 {
10474 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10475 }
10476
10477 /*
10478 * 64-bit checks.
10479 */
10480#if HC_ARCH_BITS == 64
10481 if (fLongModeGuest)
10482 {
10483 HMVMX_CHECK_BREAK(u32GuestCr0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10484 HMVMX_CHECK_BREAK(u32GuestCr4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10485 }
10486
10487 if ( !fLongModeGuest
10488 && (u32GuestCr4 & X86_CR4_PCIDE))
10489 {
10490 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10491 }
10492
10493 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10494 * 51:32 beyond the processor's physical-address width are 0. */
10495
10496 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10497 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10498 {
10499 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10500 }
10501
10502 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10503 AssertRCBreak(rc);
10504 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10505
10506 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10507 AssertRCBreak(rc);
10508 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10509#endif
10510
10511 /*
10512 * PERF_GLOBAL MSR.
10513 */
10514 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10515 {
10516 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10517 AssertRCBreak(rc);
10518 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10519 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10520 }
10521
10522 /*
10523 * PAT MSR.
10524 */
10525 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10526 {
10527 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10528 AssertRCBreak(rc);
10529 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10530 for (unsigned i = 0; i < 8; i++)
10531 {
10532 uint8_t u8Val = (u64Val & 0xff);
10533 if ( u8Val != 0 /* UC */
10534 && u8Val != 1 /* WC */
10535 && u8Val != 4 /* WT */
10536 && u8Val != 5 /* WP */
10537 && u8Val != 6 /* WB */
10538 && u8Val != 7 /* UC- */)
10539 {
10540 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10541 }
10542 u64Val >>= 8;
10543 }
10544 }
10545
10546 /*
10547 * EFER MSR.
10548 */
10549 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10550 {
10551 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10552 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10553 AssertRCBreak(rc);
10554 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10555 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10556 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10557 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10558 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10559 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10560 || !(u32GuestCr0 & X86_CR0_PG)
10561 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10562 VMX_IGS_EFER_LMA_LME_MISMATCH);
10563 }
10564
10565 /*
10566 * Segment registers.
10567 */
10568 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10569 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10570 if (!(u32Eflags & X86_EFL_VM))
10571 {
10572 /* CS */
10573 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10574 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10575 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10576 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10577 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10578 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10579 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10580 /* CS cannot be loaded with NULL in protected mode. */
10581 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10582 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10583 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10584 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10585 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10586 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10587 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10588 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10589 else
10590 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10591
10592 /* SS */
10593 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10594 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10595 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10596 if ( !(pCtx->cr0 & X86_CR0_PE)
10597 || pCtx->cs.Attr.n.u4Type == 3)
10598 {
10599 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10600 }
10601 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10602 {
10603 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10604 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10605 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10606 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10607 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10608 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10609 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10610 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10611 }
10612
10613 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxExportGuestSegmenReg(). */
10614 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10615 {
10616 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10617 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10618 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10619 || pCtx->ds.Attr.n.u4Type > 11
10620 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10621 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10622 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10623 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10624 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10625 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10626 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10627 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10628 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10629 }
10630 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10631 {
10632 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10633 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10634 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10635 || pCtx->es.Attr.n.u4Type > 11
10636 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10637 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10638 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10639 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10640 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10641 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10642 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10643 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10644 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10645 }
10646 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10647 {
10648 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10649 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10650 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10651 || pCtx->fs.Attr.n.u4Type > 11
10652 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10653 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10654 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10655 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10656 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10657 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10658 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10659 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10660 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10661 }
10662 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10663 {
10664 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10665 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10666 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10667 || pCtx->gs.Attr.n.u4Type > 11
10668 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10669 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10670 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10671 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10672 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10673 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10674 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10675 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10676 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10677 }
10678 /* 64-bit capable CPUs. */
10679#if HC_ARCH_BITS == 64
10680 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10681 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10682 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10683 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10684 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10685 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10686 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10687 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10688 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10689 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10690 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10691#endif
10692 }
10693 else
10694 {
10695 /* V86 mode checks. */
10696 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10697 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10698 {
10699 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10700 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10701 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10702 }
10703 else
10704 {
10705 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10706 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10707 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10708 }
10709
10710 /* CS */
10711 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10712 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10713 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10714 /* SS */
10715 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10716 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10717 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10718 /* DS */
10719 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10720 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10721 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10722 /* ES */
10723 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10724 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10725 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10726 /* FS */
10727 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10728 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10729 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10730 /* GS */
10731 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10732 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10733 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10734 /* 64-bit capable CPUs. */
10735#if HC_ARCH_BITS == 64
10736 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10737 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10738 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10739 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10740 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10741 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10742 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10743 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10744 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10745 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10746 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10747#endif
10748 }
10749
10750 /*
10751 * TR.
10752 */
10753 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10754 /* 64-bit capable CPUs. */
10755#if HC_ARCH_BITS == 64
10756 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10757#endif
10758 if (fLongModeGuest)
10759 {
10760 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10761 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10762 }
10763 else
10764 {
10765 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10766 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10767 VMX_IGS_TR_ATTR_TYPE_INVALID);
10768 }
10769 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10770 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10771 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10772 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10773 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10774 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10775 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10776 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10777
10778 /*
10779 * GDTR and IDTR.
10780 */
10781#if HC_ARCH_BITS == 64
10782 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10783 AssertRCBreak(rc);
10784 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10785
10786 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10787 AssertRCBreak(rc);
10788 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10789#endif
10790
10791 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10792 AssertRCBreak(rc);
10793 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10794
10795 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10796 AssertRCBreak(rc);
10797 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10798
10799 /*
10800 * Guest Non-Register State.
10801 */
10802 /* Activity State. */
10803 uint32_t u32ActivityState;
10804 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10805 AssertRCBreak(rc);
10806 HMVMX_CHECK_BREAK( !u32ActivityState
10807 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10808 VMX_IGS_ACTIVITY_STATE_INVALID);
10809 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10810 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10811 uint32_t u32IntrState;
10812 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10813 AssertRCBreak(rc);
10814 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10815 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10816 {
10817 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10818 }
10819
10820 /** @todo Activity state and injecting interrupts. Left as a todo since we
10821 * currently don't use activity states but ACTIVE. */
10822
10823 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10824 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10825
10826 /* Guest interruptibility-state. */
10827 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10828 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10829 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10830 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10831 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10832 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10833 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10834 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10835 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10836 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10837 {
10838 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10839 {
10840 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10841 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10842 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10843 }
10844 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10845 {
10846 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10847 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10848 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10849 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10850 }
10851 }
10852 /** @todo Assumes the processor is not in SMM. */
10853 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10854 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10855 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10856 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10857 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10858 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10859 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10860 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10861 {
10862 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10863 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10864 }
10865
10866 /* Pending debug exceptions. */
10867#if HC_ARCH_BITS == 64
10868 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10869 AssertRCBreak(rc);
10870 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10871 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10872 u32Val = u64Val; /* For pending debug exceptions checks below. */
10873#else
10874 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10875 AssertRCBreak(rc);
10876 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10877 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10878#endif
10879
10880 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10881 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10882 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10883 {
10884 if ( (u32Eflags & X86_EFL_TF)
10885 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10886 {
10887 /* Bit 14 is PendingDebug.BS. */
10888 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10889 }
10890 if ( !(u32Eflags & X86_EFL_TF)
10891 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10892 {
10893 /* Bit 14 is PendingDebug.BS. */
10894 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10895 }
10896 }
10897
10898 /* VMCS link pointer. */
10899 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10900 AssertRCBreak(rc);
10901 if (u64Val != UINT64_C(0xffffffffffffffff))
10902 {
10903 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10904 /** @todo Bits beyond the processor's physical-address width MBZ. */
10905 /** @todo 32-bit located in memory referenced by value of this field (as a
10906 * physical address) must contain the processor's VMCS revision ID. */
10907 /** @todo SMM checks. */
10908 }
10909
10910 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10911 * not using Nested Paging? */
10912 if ( pVM->hm.s.fNestedPaging
10913 && !fLongModeGuest
10914 && CPUMIsGuestInPAEModeEx(pCtx))
10915 {
10916 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10917 AssertRCBreak(rc);
10918 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10919
10920 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10921 AssertRCBreak(rc);
10922 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10923
10924 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10925 AssertRCBreak(rc);
10926 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10927
10928 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10929 AssertRCBreak(rc);
10930 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10931 }
10932
10933 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10934 if (uError == VMX_IGS_ERROR)
10935 uError = VMX_IGS_REASON_NOT_FOUND;
10936 } while (0);
10937
10938 pVCpu->hm.s.u32HMError = uError;
10939 return uError;
10940
10941#undef HMVMX_ERROR_BREAK
10942#undef HMVMX_CHECK_BREAK
10943}
10944
10945/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10946/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10947/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10948
10949/** @name VM-exit handlers.
10950 * @{
10951 */
10952
10953/**
10954 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10955 */
10956HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10957{
10958 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10959 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10960 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10961 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10962 return VINF_SUCCESS;
10963 return VINF_EM_RAW_INTERRUPT;
10964}
10965
10966
10967/**
10968 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10969 */
10970HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10971{
10972 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10973 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10974
10975 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10976 AssertRCReturn(rc, rc);
10977
10978 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10979 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10980 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10981 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10982
10983 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10984 {
10985 /*
10986 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10987 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10988 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10989 *
10990 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10991 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10992 */
10993 VMXDispatchHostNmi();
10994 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10995 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10996 return VINF_SUCCESS;
10997 }
10998
10999 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
11000 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
11001 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
11002 { /* likely */ }
11003 else
11004 {
11005 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
11006 rcStrictRc1 = VINF_SUCCESS;
11007 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11008 return rcStrictRc1;
11009 }
11010
11011 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
11012 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
11013 switch (uIntType)
11014 {
11015 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
11016 Assert(uVector == X86_XCPT_DB);
11017 RT_FALL_THRU();
11018 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
11019 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
11020 RT_FALL_THRU();
11021 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
11022 {
11023 /*
11024 * If there's any exception caused as a result of event injection, the resulting
11025 * secondary/final execption will be pending, we shall continue guest execution
11026 * after injecting the event. The page-fault case is complicated and we manually
11027 * handle any currently pending event in hmR0VmxExitXcptPF.
11028 */
11029 if (!pVCpu->hm.s.Event.fPending)
11030 { /* likely */ }
11031 else if (uVector != X86_XCPT_PF)
11032 {
11033 rc = VINF_SUCCESS;
11034 break;
11035 }
11036
11037 switch (uVector)
11038 {
11039 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
11040 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
11041 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11042 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11043 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11044 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11045
11046 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11047 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11048 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11049 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11050 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11051 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11052 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11053 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11054 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11055 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11056 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11057 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11058 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11059 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11060 default:
11061 {
11062 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11063 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11064 {
11065 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11066 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11067 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11068
11069 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11070 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11071 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11072 AssertRCReturn(rc, rc);
11073 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11074 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11075 0 /* GCPtrFaultAddress */);
11076 }
11077 else
11078 {
11079 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11080 pVCpu->hm.s.u32HMError = uVector;
11081 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11082 }
11083 break;
11084 }
11085 }
11086 break;
11087 }
11088
11089 default:
11090 {
11091 pVCpu->hm.s.u32HMError = uExitIntInfo;
11092 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11093 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11094 break;
11095 }
11096 }
11097 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11098 return rc;
11099}
11100
11101
11102/**
11103 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11104 */
11105HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11106{
11107 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11108
11109 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11110 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11111
11112 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11114 return VINF_SUCCESS;
11115}
11116
11117
11118/**
11119 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11120 */
11121HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11122{
11123 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11124 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11125 {
11126 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11127 HMVMX_RETURN_UNEXPECTED_EXIT();
11128 }
11129
11130 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11131
11132 /*
11133 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11134 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11135 */
11136 uint32_t fIntrState = 0;
11137 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11138 AssertRCReturn(rc, rc);
11139
11140 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11141 if ( fBlockSti
11142 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11143 {
11144 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11145 }
11146
11147 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11148 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11149
11150 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11151 return VINF_SUCCESS;
11152}
11153
11154
11155/**
11156 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11157 */
11158HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11159{
11160 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11161 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11162}
11163
11164
11165/**
11166 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11167 */
11168HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11169{
11170 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11171 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11172}
11173
11174
11175/**
11176 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11177 */
11178HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11179{
11180 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11181 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11182
11183 /*
11184 * Get the state we need and update the exit history entry.
11185 */
11186 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11187 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11188 | CPUMCTX_EXTRN_CS);
11189 AssertRCReturn(rc, rc);
11190
11191 VBOXSTRICTRC rcStrict;
11192 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11193 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11194 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11195 if (!pExitRec)
11196 {
11197 /*
11198 * Regular CPUID instruction execution.
11199 */
11200 PVM pVM = pVCpu->CTX_SUFF(pVM);
11201 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11202 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11203 {
11204 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11205 Assert(pVmxTransient->cbInstr == 2);
11206 }
11207 else
11208 {
11209 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11210 rcStrict = VERR_EM_INTERPRETER;
11211 }
11212 }
11213 else
11214 {
11215 /*
11216 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11217 */
11218 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11219 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11220 AssertRCReturn(rc2, rc2);
11221
11222 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11223 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11224
11225 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11226 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11227
11228 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11229 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11230 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11231 }
11232 return VBOXSTRICTRC_TODO(rcStrict);
11233}
11234
11235
11236/**
11237 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11238 */
11239HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11240{
11241 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11242 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11243 AssertRCReturn(rc, rc);
11244
11245 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11246 return VINF_EM_RAW_EMULATE_INSTR;
11247
11248 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11249 HMVMX_RETURN_UNEXPECTED_EXIT();
11250}
11251
11252
11253/**
11254 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11255 */
11256HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11257{
11258 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11259 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11260 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11261 AssertRCReturn(rc, rc);
11262
11263 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11264 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11265 {
11266 /* If we get a spurious VM-exit when offsetting is enabled,
11267 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11268 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11269 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11270 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11271 | HM_CHANGED_GUEST_RFLAGS);
11272 }
11273 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11274 {
11275 rcStrict = VINF_SUCCESS;
11276 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11277 }
11278 return rcStrict;
11279}
11280
11281
11282/**
11283 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11284 */
11285HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11286{
11287 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11288 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11289 | CPUMCTX_EXTRN_TSC_AUX);
11290 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11291 AssertRCReturn(rc, rc);
11292
11293 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11294 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11295 {
11296 /* If we get a spurious VM-exit when offsetting is enabled,
11297 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11298 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11299 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11300 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11301 | HM_CHANGED_GUEST_RFLAGS);
11302 }
11303 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11304 {
11305 rcStrict = VINF_SUCCESS;
11306 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11307 }
11308 return rcStrict;
11309}
11310
11311
11312/**
11313 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11314 */
11315HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11316{
11317 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11318 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11319 | CPUMCTX_EXTRN_CR0
11320 | CPUMCTX_EXTRN_RFLAGS
11321 | CPUMCTX_EXTRN_SS);
11322 AssertRCReturn(rc, rc);
11323
11324 PVM pVM = pVCpu->CTX_SUFF(pVM);
11325 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11326 if (RT_LIKELY(rc == VINF_SUCCESS))
11327 {
11328 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11329 Assert(pVmxTransient->cbInstr == 2);
11330 }
11331 else
11332 {
11333 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11334 rc = VERR_EM_INTERPRETER;
11335 }
11336 return rc;
11337}
11338
11339
11340/**
11341 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11342 */
11343HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11344{
11345 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11346
11347 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11348 if (EMAreHypercallInstructionsEnabled(pVCpu))
11349 {
11350 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11351 | CPUMCTX_EXTRN_RFLAGS
11352 | CPUMCTX_EXTRN_CR0
11353 | CPUMCTX_EXTRN_SS
11354 | CPUMCTX_EXTRN_CS
11355 | CPUMCTX_EXTRN_EFER);
11356 AssertRCReturn(rc, rc);
11357
11358 /* Perform the hypercall. */
11359 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11360 if (rcStrict == VINF_SUCCESS)
11361 {
11362 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11363 AssertRCReturn(rc, rc);
11364 }
11365 else
11366 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11367 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11368 || RT_FAILURE(rcStrict));
11369
11370 /* If the hypercall changes anything other than guest's general-purpose registers,
11371 we would need to reload the guest changed bits here before VM-entry. */
11372 }
11373 else
11374 Log4Func(("Hypercalls not enabled\n"));
11375
11376 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11377 if (RT_FAILURE(rcStrict))
11378 {
11379 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11380 rcStrict = VINF_SUCCESS;
11381 }
11382
11383 return rcStrict;
11384}
11385
11386
11387/**
11388 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11389 */
11390HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11391{
11392 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11393 PVM pVM = pVCpu->CTX_SUFF(pVM);
11394 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11395
11396 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11397 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11398 AssertRCReturn(rc, rc);
11399
11400 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11401 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11402 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11403 else
11404 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11405 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11406 return rcStrict;
11407}
11408
11409
11410/**
11411 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11412 */
11413HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11414{
11415 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11416 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11417 | CPUMCTX_EXTRN_RFLAGS
11418 | CPUMCTX_EXTRN_SS);
11419 AssertRCReturn(rc, rc);
11420
11421 PVM pVM = pVCpu->CTX_SUFF(pVM);
11422 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11423 if (RT_LIKELY(rc == VINF_SUCCESS))
11424 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11425 else
11426 {
11427 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11428 rc = VERR_EM_INTERPRETER;
11429 }
11430 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11431 return rc;
11432}
11433
11434
11435/**
11436 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11437 */
11438HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11439{
11440 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11441 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11442 | CPUMCTX_EXTRN_RFLAGS
11443 | CPUMCTX_EXTRN_SS);
11444 AssertRCReturn(rc, rc);
11445
11446 PVM pVM = pVCpu->CTX_SUFF(pVM);
11447 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11448 rc = VBOXSTRICTRC_VAL(rc2);
11449 if (RT_LIKELY( rc == VINF_SUCCESS
11450 || rc == VINF_EM_HALT))
11451 {
11452 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11453 AssertRCReturn(rc3, rc3);
11454
11455 if ( rc == VINF_EM_HALT
11456 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11457 rc = VINF_SUCCESS;
11458 }
11459 else
11460 {
11461 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11462 rc = VERR_EM_INTERPRETER;
11463 }
11464 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11465 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11466 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11467 return rc;
11468}
11469
11470
11471/**
11472 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11473 */
11474HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11475{
11476 /*
11477 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11478 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11479 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11480 * VMX root operation. If we get here, something funny is going on.
11481 *
11482 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11483 */
11484 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11485 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11486 HMVMX_RETURN_UNEXPECTED_EXIT();
11487}
11488
11489
11490/**
11491 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11492 */
11493HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11494{
11495 /*
11496 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11497 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11498 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11499 * an SMI. If we get here, something funny is going on.
11500 *
11501 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11502 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11503 */
11504 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11505 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11506 HMVMX_RETURN_UNEXPECTED_EXIT();
11507}
11508
11509
11510/**
11511 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11512 */
11513HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11514{
11515 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11516 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11517 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11518 HMVMX_RETURN_UNEXPECTED_EXIT();
11519}
11520
11521
11522/**
11523 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11524 */
11525HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11526{
11527 /*
11528 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11529 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11530 * See Intel spec. 25.3 "Other Causes of VM-exits".
11531 */
11532 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11533 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11534 HMVMX_RETURN_UNEXPECTED_EXIT();
11535}
11536
11537
11538/**
11539 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11540 * VM-exit.
11541 */
11542HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11543{
11544 /*
11545 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11546 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11547 *
11548 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11549 * See Intel spec. "23.8 Restrictions on VMX operation".
11550 */
11551 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11552 return VINF_SUCCESS;
11553}
11554
11555
11556/**
11557 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11558 * VM-exit.
11559 */
11560HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11561{
11562 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11563 return VINF_EM_RESET;
11564}
11565
11566
11567/**
11568 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11569 */
11570HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11571{
11572 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11573 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11574
11575 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11576 AssertRCReturn(rc, rc);
11577
11578 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11579 rc = VINF_SUCCESS;
11580 else
11581 rc = VINF_EM_HALT;
11582
11583 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11584 if (rc != VINF_SUCCESS)
11585 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11586 return rc;
11587}
11588
11589
11590/**
11591 * VM-exit handler for instructions that result in a \#UD exception delivered to
11592 * the guest.
11593 */
11594HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11595{
11596 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11597 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11598 return VINF_SUCCESS;
11599}
11600
11601
11602/**
11603 * VM-exit handler for expiry of the VMX preemption timer.
11604 */
11605HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11606{
11607 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11608
11609 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11610 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11611
11612 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11613 PVM pVM = pVCpu->CTX_SUFF(pVM);
11614 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11615 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11616 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11617}
11618
11619
11620/**
11621 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11622 */
11623HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11624{
11625 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11626
11627 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11628 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11629 | CPUMCTX_EXTRN_CR4);
11630 AssertRCReturn(rc, rc);
11631
11632 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11633 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11634 : HM_CHANGED_XCPT_RAISED_MASK);
11635
11636 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11637
11638 return rcStrict;
11639}
11640
11641
11642/**
11643 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11644 */
11645HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11646{
11647 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11648 /** @todo Use VM-exit instruction information. */
11649 return VERR_EM_INTERPRETER;
11650}
11651
11652
11653/**
11654 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11655 * Error VM-exit.
11656 */
11657HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11658{
11659 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11660 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11661 AssertRCReturn(rc, rc);
11662
11663 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11664 NOREF(uInvalidReason);
11665
11666#ifdef VBOX_STRICT
11667 uint32_t fIntrState;
11668 RTHCUINTREG uHCReg;
11669 uint64_t u64Val;
11670 uint32_t u32Val;
11671
11672 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11673 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11674 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11675 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11676 AssertRCReturn(rc, rc);
11677
11678 Log4(("uInvalidReason %u\n", uInvalidReason));
11679 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11680 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11681 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11682 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11683
11684 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11685 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11686 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11687 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11688 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11689 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11690 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11691 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11692 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11693 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11694 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11695 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11696
11697 hmR0DumpRegs(pVCpu, pMixedCtx);
11698#else
11699 NOREF(pVmxTransient);
11700#endif
11701
11702 return VERR_VMX_INVALID_GUEST_STATE;
11703}
11704
11705
11706/**
11707 * VM-exit handler for VM-entry failure due to an MSR-load
11708 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11709 */
11710HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11711{
11712 NOREF(pVmxTransient);
11713 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11714 HMVMX_RETURN_UNEXPECTED_EXIT();
11715}
11716
11717
11718/**
11719 * VM-exit handler for VM-entry failure due to a machine-check event
11720 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11721 */
11722HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11723{
11724 NOREF(pVmxTransient);
11725 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11726 HMVMX_RETURN_UNEXPECTED_EXIT();
11727}
11728
11729
11730/**
11731 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11732 * theory.
11733 */
11734HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11735{
11736 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11737 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11738 return VERR_VMX_UNDEFINED_EXIT_CODE;
11739}
11740
11741
11742/**
11743 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11744 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11745 * Conditional VM-exit.
11746 */
11747HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11748{
11749 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11750
11751 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11752 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11753 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11754 return VERR_EM_INTERPRETER;
11755 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11756 HMVMX_RETURN_UNEXPECTED_EXIT();
11757}
11758
11759
11760/**
11761 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11762 */
11763HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11764{
11765 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11766
11767 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11768 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11769 return VERR_EM_INTERPRETER;
11770 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11771 HMVMX_RETURN_UNEXPECTED_EXIT();
11772}
11773
11774
11775/**
11776 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11777 */
11778HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11779{
11780 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11781
11782 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. FS, GS (base) can be accessed by MSR reads. */
11783 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11784 | CPUMCTX_EXTRN_RFLAGS
11785 | CPUMCTX_EXTRN_SS
11786 | CPUMCTX_EXTRN_FS
11787 | CPUMCTX_EXTRN_GS);
11788 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11789 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11790 AssertRCReturn(rc, rc);
11791 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11792
11793#ifdef VBOX_STRICT
11794 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11795 {
11796 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11797 && pMixedCtx->ecx != MSR_K6_EFER)
11798 {
11799 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11800 pMixedCtx->ecx));
11801 HMVMX_RETURN_UNEXPECTED_EXIT();
11802 }
11803 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11804 {
11805 VMXMSREXITREAD enmRead;
11806 VMXMSREXITWRITE enmWrite;
11807 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11808 AssertRCReturn(rc2, rc2);
11809 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11810 {
11811 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11812 HMVMX_RETURN_UNEXPECTED_EXIT();
11813 }
11814 }
11815 }
11816#endif
11817
11818 PVM pVM = pVCpu->CTX_SUFF(pVM);
11819 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11820 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11821 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11822 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11823 if (RT_SUCCESS(rc))
11824 {
11825 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11826 Assert(pVmxTransient->cbInstr == 2);
11827 }
11828 return rc;
11829}
11830
11831
11832/**
11833 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11834 */
11835HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11836{
11837 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11838 PVM pVM = pVCpu->CTX_SUFF(pVM);
11839 int rc = VINF_SUCCESS;
11840
11841 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. FS, GS (base) can be accessed by MSR writes. */
11842 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11843 | CPUMCTX_EXTRN_RFLAGS
11844 | CPUMCTX_EXTRN_SS
11845 | CPUMCTX_EXTRN_FS
11846 | CPUMCTX_EXTRN_GS);
11847 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11848 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11849 AssertRCReturn(rc, rc);
11850 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11851
11852 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11853 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11854 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11855
11856 if (RT_SUCCESS(rc))
11857 {
11858 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11859
11860 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11861 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11862 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11863 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11864 {
11865 /*
11866 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11867 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11868 * EMInterpretWrmsr() changes it.
11869 */
11870 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11871 }
11872 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11873 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11874 else if (pMixedCtx->ecx == MSR_K6_EFER)
11875 {
11876 /*
11877 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11878 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11879 * the other bits as well, SCE and NXE. See @bugref{7368}.
11880 */
11881 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11882 | HM_CHANGED_VMX_ENTRY_CTLS
11883 | HM_CHANGED_VMX_EXIT_CTLS);
11884 }
11885
11886 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11887 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11888 {
11889 switch (pMixedCtx->ecx)
11890 {
11891 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11892 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11893 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11894 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11895 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11896 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11897 default:
11898 {
11899 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11900 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11901 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11902 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11903 break;
11904 }
11905 }
11906 }
11907#ifdef VBOX_STRICT
11908 else
11909 {
11910 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11911 switch (pMixedCtx->ecx)
11912 {
11913 case MSR_IA32_SYSENTER_CS:
11914 case MSR_IA32_SYSENTER_EIP:
11915 case MSR_IA32_SYSENTER_ESP:
11916 case MSR_K8_FS_BASE:
11917 case MSR_K8_GS_BASE:
11918 {
11919 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11920 HMVMX_RETURN_UNEXPECTED_EXIT();
11921 }
11922
11923 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11924 default:
11925 {
11926 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11927 {
11928 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11929 if (pMixedCtx->ecx != MSR_K6_EFER)
11930 {
11931 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11932 pMixedCtx->ecx));
11933 HMVMX_RETURN_UNEXPECTED_EXIT();
11934 }
11935 }
11936
11937 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11938 {
11939 VMXMSREXITREAD enmRead;
11940 VMXMSREXITWRITE enmWrite;
11941 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11942 AssertRCReturn(rc2, rc2);
11943 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11944 {
11945 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11946 HMVMX_RETURN_UNEXPECTED_EXIT();
11947 }
11948 }
11949 break;
11950 }
11951 }
11952 }
11953#endif /* VBOX_STRICT */
11954 }
11955 return rc;
11956}
11957
11958
11959/**
11960 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11961 */
11962HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11963{
11964 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11965 /** @todo The guest has likely hit a contended spinlock. We might want to
11966 * poke a schedule different guest VCPU. */
11967 return VINF_EM_RAW_INTERRUPT;
11968}
11969
11970
11971/**
11972 * VM-exit handler for when the TPR value is lowered below the specified
11973 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11974 */
11975HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11976{
11977 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11978 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11979
11980 /*
11981 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11982 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11983 */
11984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11985 return VINF_SUCCESS;
11986}
11987
11988
11989/**
11990 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11991 * VM-exit.
11992 *
11993 * @retval VINF_SUCCESS when guest execution can continue.
11994 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11995 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11996 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11997 * interpreter.
11998 */
11999HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12000{
12001 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12002 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
12003
12004 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12005 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12006 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12007 AssertRCReturn(rc, rc);
12008
12009 VBOXSTRICTRC rcStrict;
12010 PVM pVM = pVCpu->CTX_SUFF(pVM);
12011 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
12012 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
12013 switch (uAccessType)
12014 {
12015 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
12016 {
12017 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
12018 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12019 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
12020 AssertMsg( rcStrict == VINF_SUCCESS
12021 || rcStrict == VINF_IEM_RAISED_XCPT
12022 || rcStrict == VINF_PGM_CHANGE_MODE
12023 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12024
12025 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12026 {
12027 case 0:
12028 {
12029 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12030 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
12031 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
12032 break;
12033 }
12034
12035 case 2:
12036 {
12037 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
12038 /* Nothing to do here, CR2 it's not part of the VMCS. */
12039 break;
12040 }
12041
12042 case 3:
12043 {
12044 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
12045 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12046 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12047 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12048 break;
12049 }
12050
12051 case 4:
12052 {
12053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12054 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12055 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12056 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12057 break;
12058 }
12059
12060 case 8:
12061 {
12062 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12063 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12064 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12065 break;
12066 }
12067 default:
12068 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12069 break;
12070 }
12071 break;
12072 }
12073
12074 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12075 {
12076 Assert( !pVM->hm.s.fNestedPaging
12077 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12078 || pVCpu->hm.s.fUsingDebugLoop
12079 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12080 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12081 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12082 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12083
12084 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12085 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12086 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12087 AssertMsg( rcStrict == VINF_SUCCESS
12088 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12089#ifdef VBOX_WITH_STATISTICS
12090 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12091 {
12092 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12093 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12094 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12095 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12096 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12097 }
12098#endif
12099 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12100 VBOXSTRICTRC_VAL(rcStrict)));
12101 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12102 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12103 break;
12104 }
12105
12106 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12107 {
12108 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12109 AssertMsg( rcStrict == VINF_SUCCESS
12110 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12111
12112 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12113 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12114 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12115 break;
12116 }
12117
12118 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12119 {
12120 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12121 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12122 AssertMsg( rcStrict == VINF_SUCCESS
12123 || rcStrict == VINF_IEM_RAISED_XCPT
12124 || rcStrict == VINF_PGM_CHANGE_MODE,
12125 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12126
12127 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12128 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12129 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12130 break;
12131 }
12132
12133 default:
12134 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12135 VERR_VMX_UNEXPECTED_EXCEPTION);
12136 }
12137
12138 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12139 : HM_CHANGED_XCPT_RAISED_MASK);
12140 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12141 NOREF(pVM);
12142 return rcStrict;
12143}
12144
12145
12146/**
12147 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12148 * VM-exit.
12149 */
12150HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12151{
12152 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12153 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12154 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12155
12156 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12157 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12158 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12159 | CPUMCTX_EXTRN_SREG_MASK
12160 | CPUMCTX_EXTRN_EFER);
12161 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12162 AssertRCReturn(rc, rc);
12163
12164 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12165 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12166 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12167 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12168 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12169 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12170 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12171 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12172 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12173
12174 /*
12175 * Update exit history to see if this exit can be optimized.
12176 */
12177 VBOXSTRICTRC rcStrict;
12178 PCEMEXITREC pExitRec = NULL;
12179 if ( !fGstStepping
12180 && !fDbgStepping)
12181 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12182 !fIOString
12183 ? !fIOWrite
12184 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12185 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12186 : !fIOWrite
12187 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12188 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12189 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12190 if (!pExitRec)
12191 {
12192 /* I/O operation lookup arrays. */
12193 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12194 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12195 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12196 uint32_t const cbInstr = pVmxTransient->cbInstr;
12197 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12198 PVM pVM = pVCpu->CTX_SUFF(pVM);
12199 if (fIOString)
12200 {
12201 /*
12202 * INS/OUTS - I/O String instruction.
12203 *
12204 * Use instruction-information if available, otherwise fall back on
12205 * interpreting the instruction.
12206 */
12207 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12208 fIOWrite ? 'w' : 'r'));
12209 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12210 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12211 {
12212 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12213 AssertRCReturn(rc2, rc2);
12214 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12215 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12216 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12217 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12218 if (fIOWrite)
12219 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12220 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12221 else
12222 {
12223 /*
12224 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12225 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12226 * See Intel Instruction spec. for "INS".
12227 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12228 */
12229 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12230 }
12231 }
12232 else
12233 rcStrict = IEMExecOne(pVCpu);
12234
12235 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12236 fUpdateRipAlready = true;
12237 }
12238 else
12239 {
12240 /*
12241 * IN/OUT - I/O instruction.
12242 */
12243 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12244 fIOWrite ? 'w' : 'r'));
12245 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12246 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12247 if (fIOWrite)
12248 {
12249 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12250 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12251 }
12252 else
12253 {
12254 uint32_t u32Result = 0;
12255 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12256 if (IOM_SUCCESS(rcStrict))
12257 {
12258 /* Save result of I/O IN instr. in AL/AX/EAX. */
12259 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12260 }
12261 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12262 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12263 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12264 }
12265 }
12266
12267 if (IOM_SUCCESS(rcStrict))
12268 {
12269 if (!fUpdateRipAlready)
12270 {
12271 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12272 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12273 }
12274
12275 /*
12276 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12277 * while booting Fedora 17 64-bit guest.
12278 *
12279 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12280 */
12281 if (fIOString)
12282 {
12283 /** @todo Single-step for INS/OUTS with REP prefix? */
12284 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12285 }
12286 else if ( !fDbgStepping
12287 && fGstStepping)
12288 {
12289 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12290 AssertRCReturn(rc, rc);
12291 }
12292
12293 /*
12294 * If any I/O breakpoints are armed, we need to check if one triggered
12295 * and take appropriate action.
12296 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12297 */
12298 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12299 AssertRCReturn(rc, rc);
12300
12301 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12302 * execution engines about whether hyper BPs and such are pending. */
12303 uint32_t const uDr7 = pMixedCtx->dr[7];
12304 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12305 && X86_DR7_ANY_RW_IO(uDr7)
12306 && (pMixedCtx->cr4 & X86_CR4_DE))
12307 || DBGFBpIsHwIoArmed(pVM)))
12308 {
12309 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12310
12311 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12312 VMMRZCallRing3Disable(pVCpu);
12313 HM_DISABLE_PREEMPT();
12314
12315 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12316
12317 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12318 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12319 {
12320 /* Raise #DB. */
12321 if (fIsGuestDbgActive)
12322 ASMSetDR6(pMixedCtx->dr[6]);
12323 if (pMixedCtx->dr[7] != uDr7)
12324 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12325
12326 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12327 }
12328 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12329 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12330 else if ( rcStrict2 != VINF_SUCCESS
12331 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12332 rcStrict = rcStrict2;
12333 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12334
12335 HM_RESTORE_PREEMPT();
12336 VMMRZCallRing3Enable(pVCpu);
12337 }
12338 }
12339
12340#ifdef VBOX_STRICT
12341 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12342 Assert(!fIOWrite);
12343 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12344 Assert(fIOWrite);
12345 else
12346 {
12347# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12348 * statuses, that the VMM device and some others may return. See
12349 * IOM_SUCCESS() for guidance. */
12350 AssertMsg( RT_FAILURE(rcStrict)
12351 || rcStrict == VINF_SUCCESS
12352 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12353 || rcStrict == VINF_EM_DBG_BREAKPOINT
12354 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12355 || rcStrict == VINF_EM_RAW_TO_R3
12356 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12357# endif
12358 }
12359#endif
12360 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12361 }
12362 else
12363 {
12364 /*
12365 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12366 */
12367 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12368 AssertRCReturn(rc2, rc2);
12369 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12370 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12371 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12372 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12373 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12374 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12375
12376 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12377 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12378
12379 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12380 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12381 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12382 }
12383 return rcStrict;
12384}
12385
12386
12387/**
12388 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12389 * VM-exit.
12390 */
12391HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12392{
12393 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12394
12395 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12396 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12397 AssertRCReturn(rc, rc);
12398 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12399 {
12400 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12401 AssertRCReturn(rc, rc);
12402 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12403 {
12404 uint32_t uErrCode;
12405 RTGCUINTPTR GCPtrFaultAddress;
12406 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12407 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12408 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12409 if (fErrorCodeValid)
12410 {
12411 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12412 AssertRCReturn(rc, rc);
12413 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12414 }
12415 else
12416 uErrCode = 0;
12417
12418 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12419 && uVector == X86_XCPT_PF)
12420 GCPtrFaultAddress = pMixedCtx->cr2;
12421 else
12422 GCPtrFaultAddress = 0;
12423
12424 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12425 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12426
12427 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12428 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12429 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12430 }
12431 }
12432
12433 /* Fall back to the interpreter to emulate the task-switch. */
12434 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12435 return VERR_EM_INTERPRETER;
12436}
12437
12438
12439/**
12440 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12441 */
12442HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12443{
12444 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12445 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12446 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12447 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12448 AssertRCReturn(rc, rc);
12449 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12450 return VINF_EM_DBG_STEPPED;
12451}
12452
12453
12454/**
12455 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12456 */
12457HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12458{
12459 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12460
12461 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12462
12463 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12464 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12465 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12466 {
12467 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12468 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12469 {
12470 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12471 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12472 }
12473 }
12474 else
12475 {
12476 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12477 rcStrict1 = VINF_SUCCESS;
12478 return rcStrict1;
12479 }
12480
12481 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12482 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12483 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12484 AssertRCReturn(rc, rc);
12485
12486 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12487 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12488 VBOXSTRICTRC rcStrict2;
12489 switch (uAccessType)
12490 {
12491 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12492 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12493 {
12494 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12495 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12496 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12497
12498 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12499 GCPhys &= PAGE_BASE_GC_MASK;
12500 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12501 PVM pVM = pVCpu->CTX_SUFF(pVM);
12502 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12503 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12504
12505 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12506 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12507 CPUMCTX2CORE(pMixedCtx), GCPhys);
12508 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12509 if ( rcStrict2 == VINF_SUCCESS
12510 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12511 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12512 {
12513 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12514 | HM_CHANGED_GUEST_RSP
12515 | HM_CHANGED_GUEST_RFLAGS
12516 | HM_CHANGED_GUEST_APIC_TPR);
12517 rcStrict2 = VINF_SUCCESS;
12518 }
12519 break;
12520 }
12521
12522 default:
12523 Log4Func(("uAccessType=%#x\n", uAccessType));
12524 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12525 break;
12526 }
12527
12528 if (rcStrict2 != VINF_SUCCESS)
12529 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12530 return rcStrict2;
12531}
12532
12533
12534/**
12535 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12536 * VM-exit.
12537 */
12538HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12539{
12540 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12541
12542 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12543 if (pVmxTransient->fWasGuestDebugStateActive)
12544 {
12545 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12546 HMVMX_RETURN_UNEXPECTED_EXIT();
12547 }
12548
12549 if ( !pVCpu->hm.s.fSingleInstruction
12550 && !pVmxTransient->fWasHyperDebugStateActive)
12551 {
12552 Assert(!DBGFIsStepping(pVCpu));
12553 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12554
12555 /* Don't intercept MOV DRx any more. */
12556 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12557 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12558 AssertRCReturn(rc, rc);
12559
12560 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12561 VMMRZCallRing3Disable(pVCpu);
12562 HM_DISABLE_PREEMPT();
12563
12564 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12565 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12566 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12567
12568 HM_RESTORE_PREEMPT();
12569 VMMRZCallRing3Enable(pVCpu);
12570
12571#ifdef VBOX_WITH_STATISTICS
12572 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12573 AssertRCReturn(rc, rc);
12574 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12575 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12576 else
12577 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12578#endif
12579 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12580 return VINF_SUCCESS;
12581 }
12582
12583 /*
12584 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12585 * Update the segment registers and DR7 from the CPU.
12586 */
12587 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12588 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12589 | CPUMCTX_EXTRN_DR7);
12590 AssertRCReturn(rc, rc);
12591 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12592
12593 PVM pVM = pVCpu->CTX_SUFF(pVM);
12594 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12595 {
12596 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12597 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12598 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12599 if (RT_SUCCESS(rc))
12600 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12601 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12602 }
12603 else
12604 {
12605 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12606 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12607 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12608 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12609 }
12610
12611 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12612 if (RT_SUCCESS(rc))
12613 {
12614 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12615 AssertRCReturn(rc2, rc2);
12616 return VINF_SUCCESS;
12617 }
12618 return rc;
12619}
12620
12621
12622/**
12623 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12624 * Conditional VM-exit.
12625 */
12626HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12627{
12628 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12629 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12630
12631 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12632 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12633 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12634 {
12635 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12636 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12637 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12638 {
12639 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12640 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12641 }
12642 }
12643 else
12644 {
12645 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12646 rcStrict1 = VINF_SUCCESS;
12647 return rcStrict1;
12648 }
12649
12650 /*
12651 * Get sufficent state and update the exit history entry.
12652 */
12653 RTGCPHYS GCPhys;
12654 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12655 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12656 AssertRCReturn(rc, rc);
12657
12658 VBOXSTRICTRC rcStrict;
12659 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12660 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12661 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12662 if (!pExitRec)
12663 {
12664 /*
12665 * If we succeed, resume guest execution.
12666 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12667 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12668 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12669 * weird case. See @bugref{6043}.
12670 */
12671 PVM pVM = pVCpu->CTX_SUFF(pVM);
12672 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12673 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12674 if ( rcStrict == VINF_SUCCESS
12675 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12676 || rcStrict == VERR_PAGE_NOT_PRESENT)
12677 {
12678 /* Successfully handled MMIO operation. */
12679 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12680 | HM_CHANGED_GUEST_RSP
12681 | HM_CHANGED_GUEST_RFLAGS
12682 | HM_CHANGED_GUEST_APIC_TPR);
12683 rcStrict = VINF_SUCCESS;
12684 }
12685 }
12686 else
12687 {
12688 /*
12689 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12690 */
12691 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12692 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12693 AssertRCReturn(rc2, rc2);
12694
12695 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12696 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12697
12698 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12699 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12700
12701 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12702 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12703 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12704 }
12705 return VBOXSTRICTRC_TODO(rcStrict);
12706}
12707
12708
12709/**
12710 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12711 * VM-exit.
12712 */
12713HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12714{
12715 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12716 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12717
12718 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12719 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12720 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12721 {
12722 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12723 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12724 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12725 }
12726 else
12727 {
12728 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12729 rcStrict1 = VINF_SUCCESS;
12730 return rcStrict1;
12731 }
12732
12733 RTGCPHYS GCPhys;
12734 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12735 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12736 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12737 AssertRCReturn(rc, rc);
12738
12739 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12740 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12741
12742 RTGCUINT uErrorCode = 0;
12743 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12744 uErrorCode |= X86_TRAP_PF_ID;
12745 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12746 uErrorCode |= X86_TRAP_PF_RW;
12747 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12748 uErrorCode |= X86_TRAP_PF_P;
12749
12750 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12751
12752 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12753 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12754
12755 /* Handle the pagefault trap for the nested shadow table. */
12756 PVM pVM = pVCpu->CTX_SUFF(pVM);
12757 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12758 TRPMResetTrap(pVCpu);
12759
12760 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12761 if ( rcStrict2 == VINF_SUCCESS
12762 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12763 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12764 {
12765 /* Successfully synced our nested page tables. */
12766 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12767 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12768 | HM_CHANGED_GUEST_RSP
12769 | HM_CHANGED_GUEST_RFLAGS);
12770 return VINF_SUCCESS;
12771 }
12772
12773 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12774 return rcStrict2;
12775}
12776
12777/** @} */
12778
12779/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12780/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12781/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12782
12783/** @name VM-exit exception handlers.
12784 * @{
12785 */
12786
12787/**
12788 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12789 */
12790static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12791{
12792 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12793 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12794
12795 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12796 AssertRCReturn(rc, rc);
12797
12798 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12799 {
12800 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12801 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12802
12803 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12804 * provides VM-exit instruction length. If this causes problem later,
12805 * disassemble the instruction like it's done on AMD-V. */
12806 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12807 AssertRCReturn(rc2, rc2);
12808 return rc;
12809 }
12810
12811 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12812 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12813 return rc;
12814}
12815
12816
12817/**
12818 * VM-exit exception handler for \#BP (Breakpoint exception).
12819 */
12820static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12821{
12822 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12823 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12824
12825 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12826 AssertRCReturn(rc, rc);
12827
12828 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12829 if (rc == VINF_EM_RAW_GUEST_TRAP)
12830 {
12831 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12832 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12833 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12834 AssertRCReturn(rc, rc);
12835
12836 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12837 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12838 }
12839
12840 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12841 return rc;
12842}
12843
12844
12845/**
12846 * VM-exit exception handler for \#AC (alignment check exception).
12847 */
12848static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12849{
12850 RT_NOREF_PV(pMixedCtx);
12851 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12852
12853 /*
12854 * Re-inject it. We'll detect any nesting before getting here.
12855 */
12856 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12857 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12858 AssertRCReturn(rc, rc);
12859 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12860
12861 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12862 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12863 return VINF_SUCCESS;
12864}
12865
12866
12867/**
12868 * VM-exit exception handler for \#DB (Debug exception).
12869 */
12870static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12871{
12872 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12873 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12874
12875 /*
12876 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12877 * for processing.
12878 */
12879 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12880
12881 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12882 uint64_t uDR6 = X86_DR6_INIT_VAL;
12883 uDR6 |= ( pVmxTransient->uExitQualification
12884 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12885
12886 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12887 Log6Func(("rc=%Rrc\n", rc));
12888 if (rc == VINF_EM_RAW_GUEST_TRAP)
12889 {
12890 /*
12891 * The exception was for the guest. Update DR6, DR7.GD and
12892 * IA32_DEBUGCTL.LBR before forwarding it.
12893 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12894 */
12895 VMMRZCallRing3Disable(pVCpu);
12896 HM_DISABLE_PREEMPT();
12897
12898 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12899 pMixedCtx->dr[6] |= uDR6;
12900 if (CPUMIsGuestDebugStateActive(pVCpu))
12901 ASMSetDR6(pMixedCtx->dr[6]);
12902
12903 HM_RESTORE_PREEMPT();
12904 VMMRZCallRing3Enable(pVCpu);
12905
12906 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12907 AssertRCReturn(rc, rc);
12908
12909 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12910 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12911
12912 /* Paranoia. */
12913 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12914 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12915
12916 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12917 AssertRCReturn(rc, rc);
12918
12919 /*
12920 * Raise #DB in the guest.
12921 *
12922 * It is important to reflect exactly what the VM-exit gave us (preserving the
12923 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12924 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12925 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12926 *
12927 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12928 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12929 */
12930 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12931 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12932 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12933 AssertRCReturn(rc, rc);
12934 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12935 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12936 return VINF_SUCCESS;
12937 }
12938
12939 /*
12940 * Not a guest trap, must be a hypervisor related debug event then.
12941 * Update DR6 in case someone is interested in it.
12942 */
12943 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12944 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12945 CPUMSetHyperDR6(pVCpu, uDR6);
12946
12947 return rc;
12948}
12949
12950/**
12951 * VM-exit exception handler for \#GP (General-protection exception).
12952 *
12953 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12954 */
12955static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12956{
12957 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12958 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12959
12960 int rc;
12961 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12962 { /* likely */ }
12963 else
12964 {
12965#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12966 Assert(pVCpu->hm.s.fUsingDebugLoop);
12967#endif
12968 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12969 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12970 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12971 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12972 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12973 AssertRCReturn(rc, rc);
12974 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12975 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12976 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12977 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12978 return rc;
12979 }
12980
12981 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12982 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12983
12984 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12985 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12986 AssertRCReturn(rc, rc);
12987
12988 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12989 uint32_t cbOp = 0;
12990 PVM pVM = pVCpu->CTX_SUFF(pVM);
12991 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12992 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12993 if (RT_SUCCESS(rc))
12994 {
12995 rc = VINF_SUCCESS;
12996 Assert(cbOp == pDis->cbInstr);
12997 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12998 switch (pDis->pCurInstr->uOpcode)
12999 {
13000 case OP_CLI:
13001 {
13002 pMixedCtx->eflags.Bits.u1IF = 0;
13003 pMixedCtx->eflags.Bits.u1RF = 0;
13004 pMixedCtx->rip += pDis->cbInstr;
13005 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13006 if ( !fDbgStepping
13007 && pMixedCtx->eflags.Bits.u1TF)
13008 {
13009 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13010 AssertRCReturn(rc, rc);
13011 }
13012 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
13013 break;
13014 }
13015
13016 case OP_STI:
13017 {
13018 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
13019 pMixedCtx->eflags.Bits.u1IF = 1;
13020 pMixedCtx->eflags.Bits.u1RF = 0;
13021 pMixedCtx->rip += pDis->cbInstr;
13022 if (!fOldIF)
13023 {
13024 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
13025 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
13026 }
13027 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13028 if ( !fDbgStepping
13029 && pMixedCtx->eflags.Bits.u1TF)
13030 {
13031 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13032 AssertRCReturn(rc, rc);
13033 }
13034 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
13035 break;
13036 }
13037
13038 case OP_HLT:
13039 {
13040 rc = VINF_EM_HALT;
13041 pMixedCtx->rip += pDis->cbInstr;
13042 pMixedCtx->eflags.Bits.u1RF = 0;
13043 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
13044 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
13045 break;
13046 }
13047
13048 case OP_POPF:
13049 {
13050 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13051 uint32_t cbParm;
13052 uint32_t uMask;
13053 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13054 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13055 {
13056 cbParm = 4;
13057 uMask = 0xffffffff;
13058 }
13059 else
13060 {
13061 cbParm = 2;
13062 uMask = 0xffff;
13063 }
13064
13065 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13066 RTGCPTR GCPtrStack = 0;
13067 X86EFLAGS Eflags;
13068 Eflags.u32 = 0;
13069 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13070 &GCPtrStack);
13071 if (RT_SUCCESS(rc))
13072 {
13073 Assert(sizeof(Eflags.u32) >= cbParm);
13074 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13075 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13076 }
13077 if (RT_FAILURE(rc))
13078 {
13079 rc = VERR_EM_INTERPRETER;
13080 break;
13081 }
13082 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13083 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13084 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13085 pMixedCtx->esp += cbParm;
13086 pMixedCtx->esp &= uMask;
13087 pMixedCtx->rip += pDis->cbInstr;
13088 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13089 | HM_CHANGED_GUEST_RSP
13090 | HM_CHANGED_GUEST_RFLAGS);
13091 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13092 POPF restores EFLAGS.TF. */
13093 if ( !fDbgStepping
13094 && fGstStepping)
13095 {
13096 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13097 AssertRCReturn(rc, rc);
13098 }
13099 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13100 break;
13101 }
13102
13103 case OP_PUSHF:
13104 {
13105 uint32_t cbParm;
13106 uint32_t uMask;
13107 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13108 {
13109 cbParm = 4;
13110 uMask = 0xffffffff;
13111 }
13112 else
13113 {
13114 cbParm = 2;
13115 uMask = 0xffff;
13116 }
13117
13118 /* Get the stack pointer & push the contents of eflags onto the stack. */
13119 RTGCPTR GCPtrStack = 0;
13120 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13121 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13122 if (RT_FAILURE(rc))
13123 {
13124 rc = VERR_EM_INTERPRETER;
13125 break;
13126 }
13127 X86EFLAGS Eflags = pMixedCtx->eflags;
13128 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13129 Eflags.Bits.u1RF = 0;
13130 Eflags.Bits.u1VM = 0;
13131
13132 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13133 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13134 {
13135 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13136 rc = VERR_EM_INTERPRETER;
13137 break;
13138 }
13139 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13140 pMixedCtx->esp -= cbParm;
13141 pMixedCtx->esp &= uMask;
13142 pMixedCtx->rip += pDis->cbInstr;
13143 pMixedCtx->eflags.Bits.u1RF = 0;
13144 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13145 | HM_CHANGED_GUEST_RSP
13146 | HM_CHANGED_GUEST_RFLAGS);
13147 if ( !fDbgStepping
13148 && pMixedCtx->eflags.Bits.u1TF)
13149 {
13150 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13151 AssertRCReturn(rc, rc);
13152 }
13153 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13154 break;
13155 }
13156
13157 case OP_IRET:
13158 {
13159 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13160 * instruction reference. */
13161 RTGCPTR GCPtrStack = 0;
13162 uint32_t uMask = 0xffff;
13163 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13164 uint16_t aIretFrame[3];
13165 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13166 {
13167 rc = VERR_EM_INTERPRETER;
13168 break;
13169 }
13170 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13171 &GCPtrStack);
13172 if (RT_SUCCESS(rc))
13173 {
13174 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13175 PGMACCESSORIGIN_HM));
13176 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13177 }
13178 if (RT_FAILURE(rc))
13179 {
13180 rc = VERR_EM_INTERPRETER;
13181 break;
13182 }
13183 pMixedCtx->eip = 0;
13184 pMixedCtx->ip = aIretFrame[0];
13185 pMixedCtx->cs.Sel = aIretFrame[1];
13186 pMixedCtx->cs.ValidSel = aIretFrame[1];
13187 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13188 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13189 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13190 pMixedCtx->sp += sizeof(aIretFrame);
13191 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13192 | HM_CHANGED_GUEST_CS
13193 | HM_CHANGED_GUEST_RSP
13194 | HM_CHANGED_GUEST_RFLAGS);
13195 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13196 if ( !fDbgStepping
13197 && fGstStepping)
13198 {
13199 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13200 AssertRCReturn(rc, rc);
13201 }
13202 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13203 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13204 break;
13205 }
13206
13207 case OP_INT:
13208 {
13209 uint16_t uVector = pDis->Param1.uValue & 0xff;
13210 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13211 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13212 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13213 break;
13214 }
13215
13216 case OP_INTO:
13217 {
13218 if (pMixedCtx->eflags.Bits.u1OF)
13219 {
13220 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13221 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13222 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13223 }
13224 else
13225 {
13226 pMixedCtx->eflags.Bits.u1RF = 0;
13227 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13228 }
13229 break;
13230 }
13231
13232 default:
13233 {
13234 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13235 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13236 EMCODETYPE_SUPERVISOR);
13237 rc = VBOXSTRICTRC_VAL(rc2);
13238 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13239 /** @todo We have to set pending-debug exceptions here when the guest is
13240 * single-stepping depending on the instruction that was interpreted. */
13241 Log4Func(("#GP rc=%Rrc\n", rc));
13242 break;
13243 }
13244 }
13245 }
13246 else
13247 rc = VERR_EM_INTERPRETER;
13248
13249 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13250 ("#GP Unexpected rc=%Rrc\n", rc));
13251 return rc;
13252}
13253
13254
13255/**
13256 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13257 * the exception reported in the VMX transient structure back into the VM.
13258 *
13259 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13260 * up-to-date.
13261 */
13262static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13263{
13264 RT_NOREF_PV(pMixedCtx);
13265 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13266#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13267 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13268 ("uVector=%#x u32XcptBitmap=%#X32\n",
13269 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13270#endif
13271
13272 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13273 hmR0VmxCheckExitDueToEventDelivery(). */
13274 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13275 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13276 AssertRCReturn(rc, rc);
13277 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13278
13279#ifdef DEBUG_ramshankar
13280 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13281 | CPUMCTX_EXTRN_RIP);
13282 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13283 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13284#endif
13285
13286 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13287 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13288 return VINF_SUCCESS;
13289}
13290
13291
13292/**
13293 * VM-exit exception handler for \#PF (Page-fault exception).
13294 */
13295static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13296{
13297 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13298 PVM pVM = pVCpu->CTX_SUFF(pVM);
13299 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13300 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13301 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13302 AssertRCReturn(rc, rc);
13303
13304 if (!pVM->hm.s.fNestedPaging)
13305 { /* likely */ }
13306 else
13307 {
13308#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13309 Assert(pVCpu->hm.s.fUsingDebugLoop);
13310#endif
13311 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13312 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13313 {
13314 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13315 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13316 }
13317 else
13318 {
13319 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13320 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13321 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13322 }
13323 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13324 return rc;
13325 }
13326
13327 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13328 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13329 if (pVmxTransient->fVectoringPF)
13330 {
13331 Assert(pVCpu->hm.s.Event.fPending);
13332 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13333 }
13334
13335 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13336 AssertRCReturn(rc, rc);
13337
13338 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13339 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13340
13341 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13342 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13343 (RTGCPTR)pVmxTransient->uExitQualification);
13344
13345 Log4Func(("#PF: rc=%Rrc\n", rc));
13346 if (rc == VINF_SUCCESS)
13347 {
13348 /*
13349 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13350 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13351 */
13352 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13353 TRPMResetTrap(pVCpu);
13354 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13355 return rc;
13356 }
13357
13358 if (rc == VINF_EM_RAW_GUEST_TRAP)
13359 {
13360 if (!pVmxTransient->fVectoringDoublePF)
13361 {
13362 /* It's a guest page fault and needs to be reflected to the guest. */
13363 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13364 TRPMResetTrap(pVCpu);
13365 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13366 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13367 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13368 }
13369 else
13370 {
13371 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13372 TRPMResetTrap(pVCpu);
13373 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13374 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13375 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13376 }
13377
13378 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13379 return VINF_SUCCESS;
13380 }
13381
13382 TRPMResetTrap(pVCpu);
13383 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13384 return rc;
13385}
13386
13387/** @} */
13388
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette