VirtualBox

source: vbox/trunk/src/VBox/VMM/VMMR0/HMVMXR0.cpp@ 72810

Last change on this file since 72810 was 72810, checked in by vboxsync, 6 years ago

VMM/HMVMXR0: Naming.
  • Property svn:eol-style set to native
  • Property svn:keywords set to Author Date Id Revision
File size: 572.8 KB
Line 
1/* $Id: HMVMXR0.cpp 72810 2018-07-03 07:42:12Z vboxsync $ */
2/** @file
3 * HM VMX (Intel VT-x) - Host Context Ring-0.
4 */
5
6/*
7 * Copyright (C) 2012-2017 Oracle Corporation
8 *
9 * This file is part of VirtualBox Open Source Edition (OSE), as
10 * available from http://www.virtualbox.org. This file is free software;
11 * you can redistribute it and/or modify it under the terms of the GNU
12 * General Public License (GPL) as published by the Free Software
13 * Foundation, in version 2 as it comes in the "COPYING" file of the
14 * VirtualBox OSE distribution. VirtualBox OSE is distributed in the
15 * hope that it will be useful, but WITHOUT ANY WARRANTY of any kind.
16 */
17
18
19/*********************************************************************************************************************************
20* Header Files *
21*********************************************************************************************************************************/
22#define LOG_GROUP LOG_GROUP_HM
23#define VMCPU_INCL_CPUM_GST_CTX
24#include <iprt/x86.h>
25#include <iprt/asm-amd64-x86.h>
26#include <iprt/thread.h>
27
28#include <VBox/vmm/pdmapi.h>
29#include <VBox/vmm/dbgf.h>
30#include <VBox/vmm/iem.h>
31#include <VBox/vmm/iom.h>
32#include <VBox/vmm/selm.h>
33#include <VBox/vmm/tm.h>
34#include <VBox/vmm/gim.h>
35#include <VBox/vmm/apic.h>
36#ifdef VBOX_WITH_REM
37# include <VBox/vmm/rem.h>
38#endif
39#include "HMInternal.h"
40#include <VBox/vmm/vm.h>
41#include "HMVMXR0.h"
42#include "dtrace/VBoxVMM.h"
43
44#ifdef DEBUG_ramshankar
45# define HMVMX_ALWAYS_SAVE_GUEST_RFLAGS
46# define HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE
47# define HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
48# define HMVMX_ALWAYS_CHECK_GUEST_STATE
49# define HMVMX_ALWAYS_TRAP_ALL_XCPTS
50# define HMVMX_ALWAYS_TRAP_PF
51# define HMVMX_ALWAYS_FLUSH_TLB
52# define HMVMX_ALWAYS_SWAP_EFER
53#endif
54
55
56/*********************************************************************************************************************************
57* Defined Constants And Macros *
58*********************************************************************************************************************************/
59/** Use the function table. */
60#define HMVMX_USE_FUNCTION_TABLE
61
62/** Determine which tagged-TLB flush handler to use. */
63#define HMVMX_FLUSH_TAGGED_TLB_EPT_VPID 0
64#define HMVMX_FLUSH_TAGGED_TLB_EPT 1
65#define HMVMX_FLUSH_TAGGED_TLB_VPID 2
66#define HMVMX_FLUSH_TAGGED_TLB_NONE 3
67
68/** @name HMVMX_READ_XXX
69 * Flags to skip redundant reads of some common VMCS fields that are not part of
70 * the guest-CPU or VCPU state but are needed while handling VM-exits.
71 */
72#define HMVMX_READ_IDT_VECTORING_INFO RT_BIT_32(0)
73#define HMVMX_READ_IDT_VECTORING_ERROR_CODE RT_BIT_32(1)
74#define HMVMX_READ_EXIT_QUALIFICATION RT_BIT_32(2)
75#define HMVMX_READ_EXIT_INSTR_LEN RT_BIT_32(3)
76#define HMVMX_READ_EXIT_INTERRUPTION_INFO RT_BIT_32(4)
77#define HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE RT_BIT_32(5)
78#define HMVMX_READ_EXIT_INSTR_INFO RT_BIT_32(6)
79/** @} */
80
81/**
82 * States of the VMCS.
83 *
84 * This does not reflect all possible VMCS states but currently only those
85 * needed for maintaining the VMCS consistently even when thread-context hooks
86 * are used. Maybe later this can be extended (i.e. Nested Virtualization).
87 */
88#define HMVMX_VMCS_STATE_CLEAR RT_BIT(0)
89#define HMVMX_VMCS_STATE_ACTIVE RT_BIT(1)
90#define HMVMX_VMCS_STATE_LAUNCHED RT_BIT(2)
91
92/**
93 * Subset of the guest-CPU state that is kept by VMX R0 code while executing the
94 * guest using hardware-assisted VMX.
95 *
96 * This excludes state like GPRs (other than RSP) which are always are
97 * swapped and restored across the world-switch and also registers like EFER,
98 * MSR which cannot be modified by the guest without causing a VM-exit.
99 */
100#define HMVMX_CPUMCTX_EXTRN_ALL ( CPUMCTX_EXTRN_RIP \
101 | CPUMCTX_EXTRN_RFLAGS \
102 | CPUMCTX_EXTRN_RSP \
103 | CPUMCTX_EXTRN_SREG_MASK \
104 | CPUMCTX_EXTRN_TABLE_MASK \
105 | CPUMCTX_EXTRN_KERNEL_GS_BASE \
106 | CPUMCTX_EXTRN_SYSCALL_MSRS \
107 | CPUMCTX_EXTRN_SYSENTER_MSRS \
108 | CPUMCTX_EXTRN_TSC_AUX \
109 | CPUMCTX_EXTRN_OTHER_MSRS \
110 | CPUMCTX_EXTRN_CR0 \
111 | CPUMCTX_EXTRN_CR3 \
112 | CPUMCTX_EXTRN_CR4 \
113 | CPUMCTX_EXTRN_DR7 \
114 | CPUMCTX_EXTRN_HM_VMX_MASK)
115
116/**
117 * Exception bitmap mask for real-mode guests (real-on-v86).
118 *
119 * We need to intercept all exceptions manually except:
120 * - \#AC and \#DB are always intercepted to prevent the CPU from deadlocking
121 * due to bugs in Intel CPUs.
122 * - \#PF need not be intercepted even in real-mode if we have Nested Paging
123 * support.
124 */
125#define HMVMX_REAL_MODE_XCPT_MASK ( RT_BIT(X86_XCPT_DE) /* always: | RT_BIT(X86_XCPT_DB) */ | RT_BIT(X86_XCPT_NMI) \
126 | RT_BIT(X86_XCPT_BP) | RT_BIT(X86_XCPT_OF) | RT_BIT(X86_XCPT_BR) \
127 | RT_BIT(X86_XCPT_UD) | RT_BIT(X86_XCPT_NM) | RT_BIT(X86_XCPT_DF) \
128 | RT_BIT(X86_XCPT_CO_SEG_OVERRUN) | RT_BIT(X86_XCPT_TS) | RT_BIT(X86_XCPT_NP) \
129 | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_GP) /* RT_BIT(X86_XCPT_PF) */ \
130 | RT_BIT(X86_XCPT_MF) /* always: | RT_BIT(X86_XCPT_AC) */ | RT_BIT(X86_XCPT_MC) \
131 | RT_BIT(X86_XCPT_XF))
132
133/**
134 * Exception bitmap mask for all contributory exceptions.
135 *
136 * Page fault is deliberately excluded here as it's conditional as to whether
137 * it's contributory or benign. Page faults are handled separately.
138 */
139#define HMVMX_CONTRIBUTORY_XCPT_MASK ( RT_BIT(X86_XCPT_GP) | RT_BIT(X86_XCPT_NP) | RT_BIT(X86_XCPT_SS) | RT_BIT(X86_XCPT_TS) \
140 | RT_BIT(X86_XCPT_DE))
141
142/** Maximum VM-instruction error number. */
143#define HMVMX_INSTR_ERROR_MAX 28
144
145/** Profiling macro. */
146#ifdef HM_PROFILE_EXIT_DISPATCH
147# define HMVMX_START_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitDispatch, ed)
148# define HMVMX_STOP_EXIT_DISPATCH_PROF() STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitDispatch, ed)
149#else
150# define HMVMX_START_EXIT_DISPATCH_PROF() do { } while (0)
151# define HMVMX_STOP_EXIT_DISPATCH_PROF() do { } while (0)
152#endif
153
154/** Assert that preemption is disabled or covered by thread-context hooks. */
155#define HMVMX_ASSERT_PREEMPT_SAFE() Assert( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
156 || !RTThreadPreemptIsEnabled(NIL_RTTHREAD));
157
158/** Assert that we haven't migrated CPUs when thread-context hooks are not
159 * used. */
160#define HMVMX_ASSERT_CPU_SAFE() AssertMsg( VMMR0ThreadCtxHookIsEnabled(pVCpu) \
161 || pVCpu->hm.s.idEnteredCpu == RTMpCpuId(), \
162 ("Illegal migration! Entered on CPU %u Current %u\n", \
163 pVCpu->hm.s.idEnteredCpu, RTMpCpuId())); \
164
165/** Helper macro for VM-exit handlers called unexpectedly. */
166#define HMVMX_RETURN_UNEXPECTED_EXIT() \
167 do { \
168 pVCpu->hm.s.u32HMError = pVmxTransient->uExitReason; \
169 return VERR_VMX_UNEXPECTED_EXIT; \
170 } while (0)
171
172/** Macro for saving segment registers from VMCS into the guest-CPU
173 * context. */
174#ifdef VMX_USE_CACHED_VMCS_ACCESSES
175# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
176 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
177 VMX_VMCS_GUEST_##Sel##_BASE_CACHE_IDX, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
178#else
179# define HMVMX_IMPORT_SREG(Sel, a_pCtxSelReg) \
180 hmR0VmxImportGuestSegmentReg(pVCpu, VMX_VMCS16_GUEST_##Sel##_SEL, VMX_VMCS32_GUEST_##Sel##_LIMIT, \
181 VMX_VMCS_GUEST_##Sel##_BASE, VMX_VMCS32_GUEST_##Sel##_ACCESS_RIGHTS, (a_pCtxSelReg))
182#endif
183
184
185/*********************************************************************************************************************************
186* Structures and Typedefs *
187*********************************************************************************************************************************/
188/**
189 * VMX transient state.
190 *
191 * A state structure for holding miscellaneous information across
192 * VMX non-root operation and restored after the transition.
193 */
194typedef struct VMXTRANSIENT
195{
196 /** The host's rflags/eflags. */
197 RTCCUINTREG fEFlags;
198#if HC_ARCH_BITS == 32
199 uint32_t u32Alignment0;
200#endif
201 /** The guest's TPR value used for TPR shadowing. */
202 uint8_t u8GuestTpr;
203 /** Alignment. */
204 uint8_t abAlignment0[7];
205
206 /** The basic VM-exit reason. */
207 uint16_t uExitReason;
208 /** Alignment. */
209 uint16_t u16Alignment0;
210 /** The VM-exit interruption error code. */
211 uint32_t uExitIntErrorCode;
212 /** The VM-exit exit code qualification. */
213 uint64_t uExitQualification;
214
215 /** The VM-exit interruption-information field. */
216 uint32_t uExitIntInfo;
217 /** The VM-exit instruction-length field. */
218 uint32_t cbInstr;
219 /** The VM-exit instruction-information field. */
220 union
221 {
222 /** Plain unsigned int representation. */
223 uint32_t u;
224 /** INS and OUTS information. */
225 struct
226 {
227 uint32_t u7Reserved0 : 7;
228 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
229 uint32_t u3AddrSize : 3;
230 uint32_t u5Reserved1 : 5;
231 /** The segment register (X86_SREG_XXX). */
232 uint32_t iSegReg : 3;
233 uint32_t uReserved2 : 14;
234 } StrIo;
235 /** INVEPT, INVVPID, INVPCID information. */
236 struct
237 {
238 /** Scaling; 0=no scaling, 1=scale-by-2, 2=scale-by-4, 3=scale-by-8. */
239 uint32_t u2Scaling : 2;
240 uint32_t u5Reserved0 : 5;
241 /** The address size; 0=16-bit, 1=32-bit, 2=64-bit, rest undefined. */
242 uint32_t u3AddrSize : 3;
243 uint32_t u1Reserved0 : 1;
244 uint32_t u4Reserved0 : 4;
245 /** The segment register (X86_SREG_XXX). */
246 uint32_t iSegReg : 3;
247 /** The index register (X86_GREG_XXX). */
248 uint32_t iIdxReg : 4;
249 /** Set if index register is invalid. */
250 uint32_t fIdxRegValid : 1;
251 /** The base register (X86_GREG_XXX). */
252 uint32_t iBaseReg : 4;
253 /** Set if base register is invalid. */
254 uint32_t fBaseRegValid : 1;
255 /** Register 2 (X86_GREG_XXX). */
256 uint32_t iReg2 : 4;
257 } Inv;
258 } ExitInstrInfo;
259 /** Whether the VM-entry failed or not. */
260 bool fVMEntryFailed;
261 /** Alignment. */
262 uint8_t abAlignment1[3];
263
264 /** The VM-entry interruption-information field. */
265 uint32_t uEntryIntInfo;
266 /** The VM-entry exception error code field. */
267 uint32_t uEntryXcptErrorCode;
268 /** The VM-entry instruction length field. */
269 uint32_t cbEntryInstr;
270
271 /** IDT-vectoring information field. */
272 uint32_t uIdtVectoringInfo;
273 /** IDT-vectoring error code. */
274 uint32_t uIdtVectoringErrorCode;
275
276 /** Mask of currently read VMCS fields; HMVMX_READ_XXX. */
277 uint32_t fVmcsFieldsRead;
278
279 /** Whether the guest debug state was active at the time of VM-exit. */
280 bool fWasGuestDebugStateActive;
281 /** Whether the hyper debug state was active at the time of VM-exit. */
282 bool fWasHyperDebugStateActive;
283 /** Whether TSC-offsetting should be setup before VM-entry. */
284 bool fUpdateTscOffsettingAndPreemptTimer;
285 /** Whether the VM-exit was caused by a page-fault during delivery of a
286 * contributory exception or a page-fault. */
287 bool fVectoringDoublePF;
288 /** Whether the VM-exit was caused by a page-fault during delivery of an
289 * external interrupt or NMI. */
290 bool fVectoringPF;
291} VMXTRANSIENT;
292AssertCompileMemberAlignment(VMXTRANSIENT, uExitReason, sizeof(uint64_t));
293AssertCompileMemberAlignment(VMXTRANSIENT, uExitIntInfo, sizeof(uint64_t));
294AssertCompileMemberAlignment(VMXTRANSIENT, uEntryIntInfo, sizeof(uint64_t));
295AssertCompileMemberAlignment(VMXTRANSIENT, fWasGuestDebugStateActive, sizeof(uint64_t));
296AssertCompileMemberSize(VMXTRANSIENT, ExitInstrInfo, sizeof(uint32_t));
297/** Pointer to VMX transient state. */
298typedef VMXTRANSIENT *PVMXTRANSIENT;
299
300
301/**
302 * MSR-bitmap read permissions.
303 */
304typedef enum VMXMSREXITREAD
305{
306 /** Reading this MSR causes a VM-exit. */
307 VMXMSREXIT_INTERCEPT_READ = 0xb,
308 /** Reading this MSR does not cause a VM-exit. */
309 VMXMSREXIT_PASSTHRU_READ
310} VMXMSREXITREAD;
311/** Pointer to MSR-bitmap read permissions. */
312typedef VMXMSREXITREAD* PVMXMSREXITREAD;
313
314/**
315 * MSR-bitmap write permissions.
316 */
317typedef enum VMXMSREXITWRITE
318{
319 /** Writing to this MSR causes a VM-exit. */
320 VMXMSREXIT_INTERCEPT_WRITE = 0xd,
321 /** Writing to this MSR does not cause a VM-exit. */
322 VMXMSREXIT_PASSTHRU_WRITE
323} VMXMSREXITWRITE;
324/** Pointer to MSR-bitmap write permissions. */
325typedef VMXMSREXITWRITE* PVMXMSREXITWRITE;
326
327
328/**
329 * VMX VM-exit handler.
330 *
331 * @returns Strict VBox status code (i.e. informational status codes too).
332 * @param pVCpu The cross context virtual CPU structure.
333 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
334 * out-of-sync. Make sure to update the required
335 * fields before using them.
336 * @param pVmxTransient Pointer to the VMX-transient structure.
337 */
338#ifndef HMVMX_USE_FUNCTION_TABLE
339typedef VBOXSTRICTRC FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
340#else
341typedef DECLCALLBACK(VBOXSTRICTRC) FNVMXEXITHANDLER(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
342/** Pointer to VM-exit handler. */
343typedef FNVMXEXITHANDLER *PFNVMXEXITHANDLER;
344#endif
345
346/**
347 * VMX VM-exit handler, non-strict status code.
348 *
349 * This is generally the same as FNVMXEXITHANDLER, the NSRC bit is just FYI.
350 *
351 * @returns VBox status code, no informational status code returned.
352 * @param pVCpu The cross context virtual CPU structure.
353 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
354 * out-of-sync. Make sure to update the required
355 * fields before using them.
356 * @param pVmxTransient Pointer to the VMX-transient structure.
357 *
358 * @remarks This is not used on anything returning VERR_EM_INTERPRETER as the
359 * use of that status code will be replaced with VINF_EM_SOMETHING
360 * later when switching over to IEM.
361 */
362#ifndef HMVMX_USE_FUNCTION_TABLE
363typedef int FNVMXEXITHANDLERNSRC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
364#else
365typedef FNVMXEXITHANDLER FNVMXEXITHANDLERNSRC;
366#endif
367
368
369/*********************************************************************************************************************************
370* Internal Functions *
371*********************************************************************************************************************************/
372static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush);
373static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr);
374static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu);
375static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat);
376static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
377 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState);
378#if HC_ARCH_BITS == 32
379static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu);
380#endif
381#ifndef HMVMX_USE_FUNCTION_TABLE
382DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason);
383# define HMVMX_EXIT_DECL DECLINLINE(VBOXSTRICTRC)
384# define HMVMX_EXIT_NSRC_DECL DECLINLINE(int)
385#else
386# define HMVMX_EXIT_DECL static DECLCALLBACK(VBOXSTRICTRC)
387# define HMVMX_EXIT_NSRC_DECL HMVMX_EXIT_DECL
388#endif
389
390
391/** @name VM-exit handlers.
392 * @{
393 */
394static FNVMXEXITHANDLER hmR0VmxExitXcptOrNmi;
395static FNVMXEXITHANDLER hmR0VmxExitExtInt;
396static FNVMXEXITHANDLER hmR0VmxExitTripleFault;
397static FNVMXEXITHANDLERNSRC hmR0VmxExitInitSignal;
398static FNVMXEXITHANDLERNSRC hmR0VmxExitSipi;
399static FNVMXEXITHANDLERNSRC hmR0VmxExitIoSmi;
400static FNVMXEXITHANDLERNSRC hmR0VmxExitSmi;
401static FNVMXEXITHANDLERNSRC hmR0VmxExitIntWindow;
402static FNVMXEXITHANDLERNSRC hmR0VmxExitNmiWindow;
403static FNVMXEXITHANDLER hmR0VmxExitTaskSwitch;
404static FNVMXEXITHANDLER hmR0VmxExitCpuid;
405static FNVMXEXITHANDLER hmR0VmxExitGetsec;
406static FNVMXEXITHANDLER hmR0VmxExitHlt;
407static FNVMXEXITHANDLERNSRC hmR0VmxExitInvd;
408static FNVMXEXITHANDLER hmR0VmxExitInvlpg;
409static FNVMXEXITHANDLER hmR0VmxExitRdpmc;
410static FNVMXEXITHANDLER hmR0VmxExitVmcall;
411static FNVMXEXITHANDLER hmR0VmxExitRdtsc;
412static FNVMXEXITHANDLERNSRC hmR0VmxExitRsm;
413static FNVMXEXITHANDLERNSRC hmR0VmxExitSetPendingXcptUD;
414static FNVMXEXITHANDLER hmR0VmxExitMovCRx;
415static FNVMXEXITHANDLER hmR0VmxExitMovDRx;
416static FNVMXEXITHANDLER hmR0VmxExitIoInstr;
417static FNVMXEXITHANDLER hmR0VmxExitRdmsr;
418static FNVMXEXITHANDLER hmR0VmxExitWrmsr;
419static FNVMXEXITHANDLERNSRC hmR0VmxExitErrInvalidGuestState;
420static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMsrLoad;
421static FNVMXEXITHANDLERNSRC hmR0VmxExitErrUndefined;
422static FNVMXEXITHANDLER hmR0VmxExitMwait;
423static FNVMXEXITHANDLER hmR0VmxExitMtf;
424static FNVMXEXITHANDLER hmR0VmxExitMonitor;
425static FNVMXEXITHANDLER hmR0VmxExitPause;
426static FNVMXEXITHANDLERNSRC hmR0VmxExitErrMachineCheck;
427static FNVMXEXITHANDLERNSRC hmR0VmxExitTprBelowThreshold;
428static FNVMXEXITHANDLER hmR0VmxExitApicAccess;
429static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
430static FNVMXEXITHANDLER hmR0VmxExitXdtrAccess;
431static FNVMXEXITHANDLER hmR0VmxExitEptViolation;
432static FNVMXEXITHANDLER hmR0VmxExitEptMisconfig;
433static FNVMXEXITHANDLER hmR0VmxExitRdtscp;
434static FNVMXEXITHANDLER hmR0VmxExitPreemptTimer;
435static FNVMXEXITHANDLERNSRC hmR0VmxExitWbinvd;
436static FNVMXEXITHANDLER hmR0VmxExitXsetbv;
437static FNVMXEXITHANDLER hmR0VmxExitRdrand;
438static FNVMXEXITHANDLER hmR0VmxExitInvpcid;
439/** @} */
440
441static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
442static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
443static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
444static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
445static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
446static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
447static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient);
448static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx);
449
450
451/*********************************************************************************************************************************
452* Global Variables *
453*********************************************************************************************************************************/
454#ifdef HMVMX_USE_FUNCTION_TABLE
455
456/**
457 * VMX_EXIT dispatch table.
458 */
459static const PFNVMXEXITHANDLER g_apfnVMExitHandlers[VMX_EXIT_MAX + 1] =
460{
461 /* 00 VMX_EXIT_XCPT_OR_NMI */ hmR0VmxExitXcptOrNmi,
462 /* 01 VMX_EXIT_EXT_INT */ hmR0VmxExitExtInt,
463 /* 02 VMX_EXIT_TRIPLE_FAULT */ hmR0VmxExitTripleFault,
464 /* 03 VMX_EXIT_INIT_SIGNAL */ hmR0VmxExitInitSignal,
465 /* 04 VMX_EXIT_SIPI */ hmR0VmxExitSipi,
466 /* 05 VMX_EXIT_IO_SMI */ hmR0VmxExitIoSmi,
467 /* 06 VMX_EXIT_SMI */ hmR0VmxExitSmi,
468 /* 07 VMX_EXIT_INT_WINDOW */ hmR0VmxExitIntWindow,
469 /* 08 VMX_EXIT_NMI_WINDOW */ hmR0VmxExitNmiWindow,
470 /* 09 VMX_EXIT_TASK_SWITCH */ hmR0VmxExitTaskSwitch,
471 /* 10 VMX_EXIT_CPUID */ hmR0VmxExitCpuid,
472 /* 11 VMX_EXIT_GETSEC */ hmR0VmxExitGetsec,
473 /* 12 VMX_EXIT_HLT */ hmR0VmxExitHlt,
474 /* 13 VMX_EXIT_INVD */ hmR0VmxExitInvd,
475 /* 14 VMX_EXIT_INVLPG */ hmR0VmxExitInvlpg,
476 /* 15 VMX_EXIT_RDPMC */ hmR0VmxExitRdpmc,
477 /* 16 VMX_EXIT_RDTSC */ hmR0VmxExitRdtsc,
478 /* 17 VMX_EXIT_RSM */ hmR0VmxExitRsm,
479 /* 18 VMX_EXIT_VMCALL */ hmR0VmxExitVmcall,
480 /* 19 VMX_EXIT_VMCLEAR */ hmR0VmxExitSetPendingXcptUD,
481 /* 20 VMX_EXIT_VMLAUNCH */ hmR0VmxExitSetPendingXcptUD,
482 /* 21 VMX_EXIT_VMPTRLD */ hmR0VmxExitSetPendingXcptUD,
483 /* 22 VMX_EXIT_VMPTRST */ hmR0VmxExitSetPendingXcptUD,
484 /* 23 VMX_EXIT_VMREAD */ hmR0VmxExitSetPendingXcptUD,
485 /* 24 VMX_EXIT_VMRESUME */ hmR0VmxExitSetPendingXcptUD,
486 /* 25 VMX_EXIT_VMWRITE */ hmR0VmxExitSetPendingXcptUD,
487 /* 26 VMX_EXIT_VMXOFF */ hmR0VmxExitSetPendingXcptUD,
488 /* 27 VMX_EXIT_VMXON */ hmR0VmxExitSetPendingXcptUD,
489 /* 28 VMX_EXIT_MOV_CRX */ hmR0VmxExitMovCRx,
490 /* 29 VMX_EXIT_MOV_DRX */ hmR0VmxExitMovDRx,
491 /* 30 VMX_EXIT_IO_INSTR */ hmR0VmxExitIoInstr,
492 /* 31 VMX_EXIT_RDMSR */ hmR0VmxExitRdmsr,
493 /* 32 VMX_EXIT_WRMSR */ hmR0VmxExitWrmsr,
494 /* 33 VMX_EXIT_ERR_INVALID_GUEST_STATE */ hmR0VmxExitErrInvalidGuestState,
495 /* 34 VMX_EXIT_ERR_MSR_LOAD */ hmR0VmxExitErrMsrLoad,
496 /* 35 UNDEFINED */ hmR0VmxExitErrUndefined,
497 /* 36 VMX_EXIT_MWAIT */ hmR0VmxExitMwait,
498 /* 37 VMX_EXIT_MTF */ hmR0VmxExitMtf,
499 /* 38 UNDEFINED */ hmR0VmxExitErrUndefined,
500 /* 39 VMX_EXIT_MONITOR */ hmR0VmxExitMonitor,
501 /* 40 UNDEFINED */ hmR0VmxExitPause,
502 /* 41 VMX_EXIT_PAUSE */ hmR0VmxExitErrMachineCheck,
503 /* 42 VMX_EXIT_ERR_MACHINE_CHECK */ hmR0VmxExitErrUndefined,
504 /* 43 VMX_EXIT_TPR_BELOW_THRESHOLD */ hmR0VmxExitTprBelowThreshold,
505 /* 44 VMX_EXIT_APIC_ACCESS */ hmR0VmxExitApicAccess,
506 /* 45 UNDEFINED */ hmR0VmxExitErrUndefined,
507 /* 46 VMX_EXIT_XDTR_ACCESS */ hmR0VmxExitXdtrAccess,
508 /* 47 VMX_EXIT_TR_ACCESS */ hmR0VmxExitXdtrAccess,
509 /* 48 VMX_EXIT_EPT_VIOLATION */ hmR0VmxExitEptViolation,
510 /* 49 VMX_EXIT_EPT_MISCONFIG */ hmR0VmxExitEptMisconfig,
511 /* 50 VMX_EXIT_INVEPT */ hmR0VmxExitSetPendingXcptUD,
512 /* 51 VMX_EXIT_RDTSCP */ hmR0VmxExitRdtscp,
513 /* 52 VMX_EXIT_PREEMPT_TIMER */ hmR0VmxExitPreemptTimer,
514 /* 53 VMX_EXIT_INVVPID */ hmR0VmxExitSetPendingXcptUD,
515 /* 54 VMX_EXIT_WBINVD */ hmR0VmxExitWbinvd,
516 /* 55 VMX_EXIT_XSETBV */ hmR0VmxExitXsetbv,
517 /* 56 VMX_EXIT_APIC_WRITE */ hmR0VmxExitErrUndefined,
518 /* 57 VMX_EXIT_RDRAND */ hmR0VmxExitRdrand,
519 /* 58 VMX_EXIT_INVPCID */ hmR0VmxExitInvpcid,
520 /* 59 VMX_EXIT_VMFUNC */ hmR0VmxExitSetPendingXcptUD,
521 /* 60 VMX_EXIT_ENCLS */ hmR0VmxExitErrUndefined,
522 /* 61 VMX_EXIT_RDSEED */ hmR0VmxExitErrUndefined, /* only spurious exits, so undefined */
523 /* 62 VMX_EXIT_PML_FULL */ hmR0VmxExitErrUndefined,
524 /* 63 VMX_EXIT_XSAVES */ hmR0VmxExitSetPendingXcptUD,
525 /* 64 VMX_EXIT_XRSTORS */ hmR0VmxExitSetPendingXcptUD,
526};
527#endif /* HMVMX_USE_FUNCTION_TABLE */
528
529#ifdef VBOX_STRICT
530static const char * const g_apszVmxInstrErrors[HMVMX_INSTR_ERROR_MAX + 1] =
531{
532 /* 0 */ "(Not Used)",
533 /* 1 */ "VMCALL executed in VMX root operation.",
534 /* 2 */ "VMCLEAR with invalid physical address.",
535 /* 3 */ "VMCLEAR with VMXON pointer.",
536 /* 4 */ "VMLAUNCH with non-clear VMCS.",
537 /* 5 */ "VMRESUME with non-launched VMCS.",
538 /* 6 */ "VMRESUME after VMXOFF",
539 /* 7 */ "VM-entry with invalid control fields.",
540 /* 8 */ "VM-entry with invalid host state fields.",
541 /* 9 */ "VMPTRLD with invalid physical address.",
542 /* 10 */ "VMPTRLD with VMXON pointer.",
543 /* 11 */ "VMPTRLD with incorrect revision identifier.",
544 /* 12 */ "VMREAD/VMWRITE from/to unsupported VMCS component.",
545 /* 13 */ "VMWRITE to read-only VMCS component.",
546 /* 14 */ "(Not Used)",
547 /* 15 */ "VMXON executed in VMX root operation.",
548 /* 16 */ "VM-entry with invalid executive-VMCS pointer.",
549 /* 17 */ "VM-entry with non-launched executing VMCS.",
550 /* 18 */ "VM-entry with executive-VMCS pointer not VMXON pointer.",
551 /* 19 */ "VMCALL with non-clear VMCS.",
552 /* 20 */ "VMCALL with invalid VM-exit control fields.",
553 /* 21 */ "(Not Used)",
554 /* 22 */ "VMCALL with incorrect MSEG revision identifier.",
555 /* 23 */ "VMXOFF under dual monitor treatment of SMIs and SMM.",
556 /* 24 */ "VMCALL with invalid SMM-monitor features.",
557 /* 25 */ "VM-entry with invalid VM-execution control fields in executive VMCS.",
558 /* 26 */ "VM-entry with events blocked by MOV SS.",
559 /* 27 */ "(Not Used)",
560 /* 28 */ "Invalid operand to INVEPT/INVVPID."
561};
562#endif /* VBOX_STRICT */
563
564
565
566/**
567 * Updates the VM's last error record.
568 *
569 * If there was a VMX instruction error, reads the error data from the VMCS and
570 * updates VCPU's last error record as well.
571 *
572 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
573 * Can be NULL if @a rc is not VERR_VMX_UNABLE_TO_START_VM or
574 * VERR_VMX_INVALID_VMCS_FIELD.
575 * @param rc The error code.
576 */
577static void hmR0VmxUpdateErrorRecord(PVMCPU pVCpu, int rc)
578{
579 if ( rc == VERR_VMX_INVALID_VMCS_FIELD
580 || rc == VERR_VMX_UNABLE_TO_START_VM)
581 {
582 AssertPtrReturnVoid(pVCpu);
583 VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
584 }
585 pVCpu->CTX_SUFF(pVM)->hm.s.lLastError = rc;
586}
587
588
589/**
590 * Reads the VM-entry interruption-information field from the VMCS into the VMX
591 * transient structure.
592 *
593 * @returns VBox status code.
594 * @param pVmxTransient Pointer to the VMX transient structure.
595 *
596 * @remarks No-long-jump zone!!!
597 */
598DECLINLINE(int) hmR0VmxReadEntryIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
599{
600 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &pVmxTransient->uEntryIntInfo);
601 AssertRCReturn(rc, rc);
602 return VINF_SUCCESS;
603}
604
605#ifdef VBOX_STRICT
606/**
607 * Reads the VM-entry exception error code field from the VMCS into
608 * the VMX transient structure.
609 *
610 * @returns VBox status code.
611 * @param pVmxTransient Pointer to the VMX transient structure.
612 *
613 * @remarks No-long-jump zone!!!
614 */
615DECLINLINE(int) hmR0VmxReadEntryXcptErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
616{
617 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &pVmxTransient->uEntryXcptErrorCode);
618 AssertRCReturn(rc, rc);
619 return VINF_SUCCESS;
620}
621
622
623/**
624 * Reads the VM-entry exception error code field from the VMCS into
625 * the VMX transient structure.
626 *
627 * @returns VBox status code.
628 * @param pVmxTransient Pointer to the VMX transient structure.
629 *
630 * @remarks No-long-jump zone!!!
631 */
632DECLINLINE(int) hmR0VmxReadEntryInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
633{
634 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &pVmxTransient->cbEntryInstr);
635 AssertRCReturn(rc, rc);
636 return VINF_SUCCESS;
637}
638#endif /* VBOX_STRICT */
639
640
641/**
642 * Reads the VM-exit interruption-information field from the VMCS into the VMX
643 * transient structure.
644 *
645 * @returns VBox status code.
646 * @param pVmxTransient Pointer to the VMX transient structure.
647 */
648DECLINLINE(int) hmR0VmxReadExitIntInfoVmcs(PVMXTRANSIENT pVmxTransient)
649{
650 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_INFO))
651 {
652 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_INFO, &pVmxTransient->uExitIntInfo);
653 AssertRCReturn(rc,rc);
654 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_INFO;
655 }
656 return VINF_SUCCESS;
657}
658
659
660/**
661 * Reads the VM-exit interruption error code from the VMCS into the VMX
662 * transient structure.
663 *
664 * @returns VBox status code.
665 * @param pVmxTransient Pointer to the VMX transient structure.
666 */
667DECLINLINE(int) hmR0VmxReadExitIntErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
668{
669 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE))
670 {
671 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INTERRUPTION_ERROR_CODE, &pVmxTransient->uExitIntErrorCode);
672 AssertRCReturn(rc, rc);
673 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INTERRUPTION_ERROR_CODE;
674 }
675 return VINF_SUCCESS;
676}
677
678
679/**
680 * Reads the VM-exit instruction length field from the VMCS into the VMX
681 * transient structure.
682 *
683 * @returns VBox status code.
684 * @param pVmxTransient Pointer to the VMX transient structure.
685 */
686DECLINLINE(int) hmR0VmxReadExitInstrLenVmcs(PVMXTRANSIENT pVmxTransient)
687{
688 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_LEN))
689 {
690 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_LENGTH, &pVmxTransient->cbInstr);
691 AssertRCReturn(rc, rc);
692 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_LEN;
693 }
694 return VINF_SUCCESS;
695}
696
697
698/**
699 * Reads the VM-exit instruction-information field from the VMCS into
700 * the VMX transient structure.
701 *
702 * @returns VBox status code.
703 * @param pVmxTransient Pointer to the VMX transient structure.
704 */
705DECLINLINE(int) hmR0VmxReadExitInstrInfoVmcs(PVMXTRANSIENT pVmxTransient)
706{
707 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_INSTR_INFO))
708 {
709 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_INSTR_INFO, &pVmxTransient->ExitInstrInfo.u);
710 AssertRCReturn(rc, rc);
711 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_INSTR_INFO;
712 }
713 return VINF_SUCCESS;
714}
715
716
717/**
718 * Reads the exit code qualification from the VMCS into the VMX transient
719 * structure.
720 *
721 * @returns VBox status code.
722 * @param pVCpu The cross context virtual CPU structure of the
723 * calling EMT. (Required for the VMCS cache case.)
724 * @param pVmxTransient Pointer to the VMX transient structure.
725 */
726DECLINLINE(int) hmR0VmxReadExitQualificationVmcs(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient)
727{
728 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_EXIT_QUALIFICATION))
729 {
730 int rc = VMXReadVmcsGstN(VMX_VMCS_RO_EXIT_QUALIFICATION, &pVmxTransient->uExitQualification); NOREF(pVCpu);
731 AssertRCReturn(rc, rc);
732 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_EXIT_QUALIFICATION;
733 }
734 return VINF_SUCCESS;
735}
736
737
738/**
739 * Reads the IDT-vectoring information field from the VMCS into the VMX
740 * transient structure.
741 *
742 * @returns VBox status code.
743 * @param pVmxTransient Pointer to the VMX transient structure.
744 *
745 * @remarks No-long-jump zone!!!
746 */
747DECLINLINE(int) hmR0VmxReadIdtVectoringInfoVmcs(PVMXTRANSIENT pVmxTransient)
748{
749 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_INFO))
750 {
751 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_INFO, &pVmxTransient->uIdtVectoringInfo);
752 AssertRCReturn(rc, rc);
753 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_INFO;
754 }
755 return VINF_SUCCESS;
756}
757
758
759/**
760 * Reads the IDT-vectoring error code from the VMCS into the VMX
761 * transient structure.
762 *
763 * @returns VBox status code.
764 * @param pVmxTransient Pointer to the VMX transient structure.
765 */
766DECLINLINE(int) hmR0VmxReadIdtVectoringErrorCodeVmcs(PVMXTRANSIENT pVmxTransient)
767{
768 if (!(pVmxTransient->fVmcsFieldsRead & HMVMX_READ_IDT_VECTORING_ERROR_CODE))
769 {
770 int rc = VMXReadVmcs32(VMX_VMCS32_RO_IDT_VECTORING_ERROR_CODE, &pVmxTransient->uIdtVectoringErrorCode);
771 AssertRCReturn(rc, rc);
772 pVmxTransient->fVmcsFieldsRead |= HMVMX_READ_IDT_VECTORING_ERROR_CODE;
773 }
774 return VINF_SUCCESS;
775}
776
777
778/**
779 * Enters VMX root mode operation on the current CPU.
780 *
781 * @returns VBox status code.
782 * @param pVM The cross context VM structure. Can be
783 * NULL, after a resume.
784 * @param HCPhysCpuPage Physical address of the VMXON region.
785 * @param pvCpuPage Pointer to the VMXON region.
786 */
787static int hmR0VmxEnterRootMode(PVM pVM, RTHCPHYS HCPhysCpuPage, void *pvCpuPage)
788{
789 Assert(HCPhysCpuPage && HCPhysCpuPage != NIL_RTHCPHYS);
790 Assert(RT_ALIGN_T(HCPhysCpuPage, _4K, RTHCPHYS) == HCPhysCpuPage);
791 Assert(pvCpuPage);
792 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
793
794 if (pVM)
795 {
796 /* Write the VMCS revision dword to the VMXON region. */
797 *(uint32_t *)pvCpuPage = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
798 }
799
800 /* Paranoid: Disable interrupts as, in theory, interrupt handlers might mess with CR4. */
801 RTCCUINTREG fEFlags = ASMIntDisableFlags();
802
803 /* Enable the VMX bit in CR4 if necessary. */
804 RTCCUINTREG uOldCr4 = SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
805
806 /* Enter VMX root mode. */
807 int rc = VMXEnable(HCPhysCpuPage);
808 if (RT_FAILURE(rc))
809 {
810 if (!(uOldCr4 & X86_CR4_VMXE))
811 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
812
813 if (pVM)
814 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
815 }
816
817 /* Restore interrupts. */
818 ASMSetFlags(fEFlags);
819 return rc;
820}
821
822
823/**
824 * Exits VMX root mode operation on the current CPU.
825 *
826 * @returns VBox status code.
827 */
828static int hmR0VmxLeaveRootMode(void)
829{
830 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
831
832 /* Paranoid: Disable interrupts as, in theory, interrupts handlers might mess with CR4. */
833 RTCCUINTREG fEFlags = ASMIntDisableFlags();
834
835 /* If we're for some reason not in VMX root mode, then don't leave it. */
836 RTCCUINTREG uHostCR4 = ASMGetCR4();
837
838 int rc;
839 if (uHostCR4 & X86_CR4_VMXE)
840 {
841 /* Exit VMX root mode and clear the VMX bit in CR4. */
842 VMXDisable();
843 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
844 rc = VINF_SUCCESS;
845 }
846 else
847 rc = VERR_VMX_NOT_IN_VMX_ROOT_MODE;
848
849 /* Restore interrupts. */
850 ASMSetFlags(fEFlags);
851 return rc;
852}
853
854
855/**
856 * Allocates and maps one physically contiguous page. The allocated page is
857 * zero'd out. (Used by various VT-x structures).
858 *
859 * @returns IPRT status code.
860 * @param pMemObj Pointer to the ring-0 memory object.
861 * @param ppVirt Where to store the virtual address of the
862 * allocation.
863 * @param pHCPhys Where to store the physical address of the
864 * allocation.
865 */
866static int hmR0VmxPageAllocZ(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
867{
868 AssertPtrReturn(pMemObj, VERR_INVALID_PARAMETER);
869 AssertPtrReturn(ppVirt, VERR_INVALID_PARAMETER);
870 AssertPtrReturn(pHCPhys, VERR_INVALID_PARAMETER);
871
872 int rc = RTR0MemObjAllocCont(pMemObj, PAGE_SIZE, false /* fExecutable */);
873 if (RT_FAILURE(rc))
874 return rc;
875 *ppVirt = RTR0MemObjAddress(*pMemObj);
876 *pHCPhys = RTR0MemObjGetPagePhysAddr(*pMemObj, 0 /* iPage */);
877 ASMMemZero32(*ppVirt, PAGE_SIZE);
878 return VINF_SUCCESS;
879}
880
881
882/**
883 * Frees and unmaps an allocated physical page.
884 *
885 * @param pMemObj Pointer to the ring-0 memory object.
886 * @param ppVirt Where to re-initialize the virtual address of
887 * allocation as 0.
888 * @param pHCPhys Where to re-initialize the physical address of the
889 * allocation as 0.
890 */
891static void hmR0VmxPageFree(PRTR0MEMOBJ pMemObj, PRTR0PTR ppVirt, PRTHCPHYS pHCPhys)
892{
893 AssertPtr(pMemObj);
894 AssertPtr(ppVirt);
895 AssertPtr(pHCPhys);
896 if (*pMemObj != NIL_RTR0MEMOBJ)
897 {
898 int rc = RTR0MemObjFree(*pMemObj, true /* fFreeMappings */);
899 AssertRC(rc);
900 *pMemObj = NIL_RTR0MEMOBJ;
901 *ppVirt = 0;
902 *pHCPhys = 0;
903 }
904}
905
906
907/**
908 * Worker function to free VT-x related structures.
909 *
910 * @returns IPRT status code.
911 * @param pVM The cross context VM structure.
912 */
913static void hmR0VmxStructsFree(PVM pVM)
914{
915 for (VMCPUID i = 0; i < pVM->cCpus; i++)
916 {
917 PVMCPU pVCpu = &pVM->aCpus[i];
918 AssertPtr(pVCpu);
919
920 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
921 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
922
923 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
924 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap, &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
925
926 hmR0VmxPageFree(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
927 }
928
929 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess, &pVM->hm.s.vmx.HCPhysApicAccess);
930#ifdef VBOX_WITH_CRASHDUMP_MAGIC
931 hmR0VmxPageFree(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
932#endif
933}
934
935
936/**
937 * Worker function to allocate VT-x related VM structures.
938 *
939 * @returns IPRT status code.
940 * @param pVM The cross context VM structure.
941 */
942static int hmR0VmxStructsAlloc(PVM pVM)
943{
944 /*
945 * Initialize members up-front so we can cleanup properly on allocation failure.
946 */
947#define VMXLOCAL_INIT_VM_MEMOBJ(a_Name, a_VirtPrefix) \
948 pVM->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
949 pVM->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
950 pVM->hm.s.vmx.HCPhys##a_Name = 0;
951
952#define VMXLOCAL_INIT_VMCPU_MEMOBJ(a_Name, a_VirtPrefix) \
953 pVCpu->hm.s.vmx.hMemObj##a_Name = NIL_RTR0MEMOBJ; \
954 pVCpu->hm.s.vmx.a_VirtPrefix##a_Name = 0; \
955 pVCpu->hm.s.vmx.HCPhys##a_Name = 0;
956
957#ifdef VBOX_WITH_CRASHDUMP_MAGIC
958 VMXLOCAL_INIT_VM_MEMOBJ(Scratch, pv);
959#endif
960 VMXLOCAL_INIT_VM_MEMOBJ(ApicAccess, pb);
961
962 AssertCompile(sizeof(VMCPUID) == sizeof(pVM->cCpus));
963 for (VMCPUID i = 0; i < pVM->cCpus; i++)
964 {
965 PVMCPU pVCpu = &pVM->aCpus[i];
966 VMXLOCAL_INIT_VMCPU_MEMOBJ(Vmcs, pv);
967 VMXLOCAL_INIT_VMCPU_MEMOBJ(MsrBitmap, pv);
968 VMXLOCAL_INIT_VMCPU_MEMOBJ(GuestMsr, pv);
969 VMXLOCAL_INIT_VMCPU_MEMOBJ(HostMsr, pv);
970 }
971#undef VMXLOCAL_INIT_VMCPU_MEMOBJ
972#undef VMXLOCAL_INIT_VM_MEMOBJ
973
974 /* The VMCS size cannot be more than 4096 bytes. See Intel spec. Appendix A.1 "Basic VMX Information". */
975 AssertReturnStmt(MSR_IA32_VMX_BASIC_INFO_VMCS_SIZE(pVM->hm.s.vmx.Msrs.u64BasicInfo) <= PAGE_SIZE,
976 (&pVM->aCpus[0])->hm.s.u32HMError = VMX_UFC_INVALID_VMCS_SIZE,
977 VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO);
978
979 /*
980 * Allocate all the VT-x structures.
981 */
982 int rc = VINF_SUCCESS;
983#ifdef VBOX_WITH_CRASHDUMP_MAGIC
984 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjScratch, &pVM->hm.s.vmx.pbScratch, &pVM->hm.s.vmx.HCPhysScratch);
985 if (RT_FAILURE(rc))
986 goto cleanup;
987 strcpy((char *)pVM->hm.s.vmx.pbScratch, "SCRATCH Magic");
988 *(uint64_t *)(pVM->hm.s.vmx.pbScratch + 16) = UINT64_C(0xdeadbeefdeadbeef);
989#endif
990
991 /* Allocate the APIC-access page for trapping APIC accesses from the guest. */
992 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
993 {
994 rc = hmR0VmxPageAllocZ(&pVM->hm.s.vmx.hMemObjApicAccess, (PRTR0PTR)&pVM->hm.s.vmx.pbApicAccess,
995 &pVM->hm.s.vmx.HCPhysApicAccess);
996 if (RT_FAILURE(rc))
997 goto cleanup;
998 }
999
1000 /*
1001 * Initialize per-VCPU VT-x structures.
1002 */
1003 for (VMCPUID i = 0; i < pVM->cCpus; i++)
1004 {
1005 PVMCPU pVCpu = &pVM->aCpus[i];
1006 AssertPtr(pVCpu);
1007
1008 /* Allocate the VM control structure (VMCS). */
1009 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjVmcs, &pVCpu->hm.s.vmx.pvVmcs, &pVCpu->hm.s.vmx.HCPhysVmcs);
1010 if (RT_FAILURE(rc))
1011 goto cleanup;
1012
1013 /* Get the allocated virtual-APIC page from the APIC device for transparent TPR accesses. */
1014 if ( PDMHasApic(pVM)
1015 && (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW))
1016 {
1017 rc = APICGetApicPageForCpu(pVCpu, &pVCpu->hm.s.vmx.HCPhysVirtApic, (PRTR0PTR)&pVCpu->hm.s.vmx.pbVirtApic,
1018 NULL /* pR3Ptr */, NULL /* pRCPtr */);
1019 if (RT_FAILURE(rc))
1020 goto cleanup;
1021 }
1022
1023 /*
1024 * Allocate the MSR-bitmap if supported by the CPU. The MSR-bitmap is for
1025 * transparent accesses of specific MSRs.
1026 *
1027 * If the condition for enabling MSR bitmaps changes here, don't forget to
1028 * update HMAreMsrBitmapsAvailable().
1029 */
1030 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1031 {
1032 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjMsrBitmap, &pVCpu->hm.s.vmx.pvMsrBitmap,
1033 &pVCpu->hm.s.vmx.HCPhysMsrBitmap);
1034 if (RT_FAILURE(rc))
1035 goto cleanup;
1036 ASMMemFill32(pVCpu->hm.s.vmx.pvMsrBitmap, PAGE_SIZE, UINT32_C(0xffffffff));
1037 }
1038
1039 /* Allocate the VM-entry MSR-load and VM-exit MSR-store page for the guest MSRs. */
1040 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjGuestMsr, &pVCpu->hm.s.vmx.pvGuestMsr, &pVCpu->hm.s.vmx.HCPhysGuestMsr);
1041 if (RT_FAILURE(rc))
1042 goto cleanup;
1043
1044 /* Allocate the VM-exit MSR-load page for the host MSRs. */
1045 rc = hmR0VmxPageAllocZ(&pVCpu->hm.s.vmx.hMemObjHostMsr, &pVCpu->hm.s.vmx.pvHostMsr, &pVCpu->hm.s.vmx.HCPhysHostMsr);
1046 if (RT_FAILURE(rc))
1047 goto cleanup;
1048 }
1049
1050 return VINF_SUCCESS;
1051
1052cleanup:
1053 hmR0VmxStructsFree(pVM);
1054 return rc;
1055}
1056
1057
1058/**
1059 * Does global VT-x initialization (called during module initialization).
1060 *
1061 * @returns VBox status code.
1062 */
1063VMMR0DECL(int) VMXR0GlobalInit(void)
1064{
1065#ifdef HMVMX_USE_FUNCTION_TABLE
1066 AssertCompile(VMX_EXIT_MAX + 1 == RT_ELEMENTS(g_apfnVMExitHandlers));
1067# ifdef VBOX_STRICT
1068 for (unsigned i = 0; i < RT_ELEMENTS(g_apfnVMExitHandlers); i++)
1069 Assert(g_apfnVMExitHandlers[i]);
1070# endif
1071#endif
1072 return VINF_SUCCESS;
1073}
1074
1075
1076/**
1077 * Does global VT-x termination (called during module termination).
1078 */
1079VMMR0DECL(void) VMXR0GlobalTerm()
1080{
1081 /* Nothing to do currently. */
1082}
1083
1084
1085/**
1086 * Sets up and activates VT-x on the current CPU.
1087 *
1088 * @returns VBox status code.
1089 * @param pHostCpu Pointer to the global CPU info struct.
1090 * @param pVM The cross context VM structure. Can be
1091 * NULL after a host resume operation.
1092 * @param pvCpuPage Pointer to the VMXON region (can be NULL if @a
1093 * fEnabledByHost is @c true).
1094 * @param HCPhysCpuPage Physical address of the VMXON region (can be 0 if
1095 * @a fEnabledByHost is @c true).
1096 * @param fEnabledByHost Set if SUPR0EnableVTx() or similar was used to
1097 * enable VT-x on the host.
1098 * @param pvMsrs Opaque pointer to VMXMSRS struct.
1099 */
1100VMMR0DECL(int) VMXR0EnableCpu(PHMGLOBALCPUINFO pHostCpu, PVM pVM, void *pvCpuPage, RTHCPHYS HCPhysCpuPage, bool fEnabledByHost,
1101 void *pvMsrs)
1102{
1103 Assert(pHostCpu);
1104 Assert(pvMsrs);
1105 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1106
1107 /* Enable VT-x if it's not already enabled by the host. */
1108 if (!fEnabledByHost)
1109 {
1110 int rc = hmR0VmxEnterRootMode(pVM, HCPhysCpuPage, pvCpuPage);
1111 if (RT_FAILURE(rc))
1112 return rc;
1113 }
1114
1115 /*
1116 * Flush all EPT tagged-TLB entries (in case VirtualBox or any other hypervisor have been
1117 * using EPTPs) so we don't retain any stale guest-physical mappings which won't get
1118 * invalidated when flushing by VPID.
1119 */
1120 PVMXMSRS pMsrs = (PVMXMSRS)pvMsrs;
1121 if (pMsrs->u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
1122 {
1123 hmR0VmxFlushEpt(NULL /* pVCpu */, VMXFLUSHEPT_ALL_CONTEXTS);
1124 pHostCpu->fFlushAsidBeforeUse = false;
1125 }
1126 else
1127 pHostCpu->fFlushAsidBeforeUse = true;
1128
1129 /* Ensure each VCPU scheduled on this CPU gets a new VPID on resume. See @bugref{6255}. */
1130 ++pHostCpu->cTlbFlushes;
1131
1132 return VINF_SUCCESS;
1133}
1134
1135
1136/**
1137 * Deactivates VT-x on the current CPU.
1138 *
1139 * @returns VBox status code.
1140 * @param pHostCpu Pointer to the global CPU info struct.
1141 * @param pvCpuPage Pointer to the VMXON region.
1142 * @param HCPhysCpuPage Physical address of the VMXON region.
1143 *
1144 * @remarks This function should never be called when SUPR0EnableVTx() or
1145 * similar was used to enable VT-x on the host.
1146 */
1147VMMR0DECL(int) VMXR0DisableCpu(PHMGLOBALCPUINFO pHostCpu, void *pvCpuPage, RTHCPHYS HCPhysCpuPage)
1148{
1149 RT_NOREF3(pHostCpu, pvCpuPage, HCPhysCpuPage);
1150
1151 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1152 return hmR0VmxLeaveRootMode();
1153}
1154
1155
1156/**
1157 * Sets the permission bits for the specified MSR in the MSR bitmap.
1158 *
1159 * @param pVCpu The cross context virtual CPU structure.
1160 * @param uMsr The MSR value.
1161 * @param enmRead Whether reading this MSR causes a VM-exit.
1162 * @param enmWrite Whether writing this MSR causes a VM-exit.
1163 */
1164static void hmR0VmxSetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, VMXMSREXITREAD enmRead, VMXMSREXITWRITE enmWrite)
1165{
1166 int32_t iBit;
1167 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1168
1169 /*
1170 * Layout:
1171 * 0x000 - 0x3ff - Low MSR read bits
1172 * 0x400 - 0x7ff - High MSR read bits
1173 * 0x800 - 0xbff - Low MSR write bits
1174 * 0xc00 - 0xfff - High MSR write bits
1175 */
1176 if (uMsr <= 0x00001FFF)
1177 iBit = uMsr;
1178 else if (uMsr - UINT32_C(0xC0000000) <= UINT32_C(0x00001FFF))
1179 {
1180 iBit = uMsr - UINT32_C(0xC0000000);
1181 pbMsrBitmap += 0x400;
1182 }
1183 else
1184 AssertMsgFailedReturnVoid(("hmR0VmxSetMsrPermission: Invalid MSR %#RX32\n", uMsr));
1185
1186 Assert(iBit <= 0x1fff);
1187 if (enmRead == VMXMSREXIT_INTERCEPT_READ)
1188 ASMBitSet(pbMsrBitmap, iBit);
1189 else
1190 ASMBitClear(pbMsrBitmap, iBit);
1191
1192 if (enmWrite == VMXMSREXIT_INTERCEPT_WRITE)
1193 ASMBitSet(pbMsrBitmap + 0x800, iBit);
1194 else
1195 ASMBitClear(pbMsrBitmap + 0x800, iBit);
1196}
1197
1198
1199#ifdef VBOX_STRICT
1200/**
1201 * Gets the permission bits for the specified MSR in the MSR bitmap.
1202 *
1203 * @returns VBox status code.
1204 * @retval VINF_SUCCESS if the specified MSR is found.
1205 * @retval VERR_NOT_FOUND if the specified MSR is not found.
1206 * @retval VERR_NOT_SUPPORTED if VT-x doesn't allow the MSR.
1207 *
1208 * @param pVCpu The cross context virtual CPU structure.
1209 * @param uMsr The MSR.
1210 * @param penmRead Where to store the read permissions.
1211 * @param penmWrite Where to store the write permissions.
1212 */
1213static int hmR0VmxGetMsrPermission(PVMCPU pVCpu, uint32_t uMsr, PVMXMSREXITREAD penmRead, PVMXMSREXITWRITE penmWrite)
1214{
1215 AssertPtrReturn(penmRead, VERR_INVALID_PARAMETER);
1216 AssertPtrReturn(penmWrite, VERR_INVALID_PARAMETER);
1217 int32_t iBit;
1218 uint8_t *pbMsrBitmap = (uint8_t *)pVCpu->hm.s.vmx.pvMsrBitmap;
1219
1220 /* See hmR0VmxSetMsrPermission() for the layout. */
1221 if (uMsr <= 0x00001FFF)
1222 iBit = uMsr;
1223 else if ( uMsr >= 0xC0000000
1224 && uMsr <= 0xC0001FFF)
1225 {
1226 iBit = (uMsr - 0xC0000000);
1227 pbMsrBitmap += 0x400;
1228 }
1229 else
1230 AssertMsgFailedReturn(("hmR0VmxGetMsrPermission: Invalid MSR %#RX32\n", uMsr), VERR_NOT_SUPPORTED);
1231
1232 Assert(iBit <= 0x1fff);
1233 if (ASMBitTest(pbMsrBitmap, iBit))
1234 *penmRead = VMXMSREXIT_INTERCEPT_READ;
1235 else
1236 *penmRead = VMXMSREXIT_PASSTHRU_READ;
1237
1238 if (ASMBitTest(pbMsrBitmap + 0x800, iBit))
1239 *penmWrite = VMXMSREXIT_INTERCEPT_WRITE;
1240 else
1241 *penmWrite = VMXMSREXIT_PASSTHRU_WRITE;
1242 return VINF_SUCCESS;
1243}
1244#endif /* VBOX_STRICT */
1245
1246
1247/**
1248 * Updates the VMCS with the number of effective MSRs in the auto-load/store MSR
1249 * area.
1250 *
1251 * @returns VBox status code.
1252 * @param pVCpu The cross context virtual CPU structure.
1253 * @param cMsrs The number of MSRs.
1254 */
1255static int hmR0VmxSetAutoLoadStoreMsrCount(PVMCPU pVCpu, uint32_t cMsrs)
1256{
1257 /* Shouldn't ever happen but there -is- a number. We're well within the recommended 512. */
1258 uint32_t const cMaxSupportedMsrs = MSR_IA32_VMX_MISC_MAX_MSR(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.u64Misc);
1259 if (RT_UNLIKELY(cMsrs > cMaxSupportedMsrs))
1260 {
1261 LogRel(("CPU auto-load/store MSR count in VMCS exceeded cMsrs=%u Supported=%u.\n", cMsrs, cMaxSupportedMsrs));
1262 pVCpu->hm.s.u32HMError = VMX_UFC_INSUFFICIENT_GUEST_MSR_STORAGE;
1263 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
1264 }
1265
1266 /* Update number of guest MSRs to load/store across the world-switch. */
1267 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, cMsrs);
1268 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, cMsrs);
1269
1270 /* Update number of host MSRs to load after the world-switch. Identical to guest-MSR count as it's always paired. */
1271 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, cMsrs);
1272 AssertRCReturn(rc, rc);
1273
1274 /* Update the VCPU's copy of the MSR count. */
1275 pVCpu->hm.s.vmx.cMsrs = cMsrs;
1276
1277 return VINF_SUCCESS;
1278}
1279
1280
1281/**
1282 * Adds a new (or updates the value of an existing) guest/host MSR
1283 * pair to be swapped during the world-switch as part of the
1284 * auto-load/store MSR area in the VMCS.
1285 *
1286 * @returns VBox status code.
1287 * @param pVCpu The cross context virtual CPU structure.
1288 * @param uMsr The MSR.
1289 * @param uGuestMsrValue Value of the guest MSR.
1290 * @param fUpdateHostMsr Whether to update the value of the host MSR if
1291 * necessary.
1292 * @param pfAddedAndUpdated Where to store whether the MSR was added -and-
1293 * its value was updated. Optional, can be NULL.
1294 */
1295static int hmR0VmxAddAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr, uint64_t uGuestMsrValue, bool fUpdateHostMsr,
1296 bool *pfAddedAndUpdated)
1297{
1298 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1299 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1300 uint32_t i;
1301 for (i = 0; i < cMsrs; i++)
1302 {
1303 if (pGuestMsr->u32Msr == uMsr)
1304 break;
1305 pGuestMsr++;
1306 }
1307
1308 bool fAdded = false;
1309 if (i == cMsrs)
1310 {
1311 ++cMsrs;
1312 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1313 AssertMsgRCReturn(rc, ("hmR0VmxAddAutoLoadStoreMsr: Insufficient space to add MSR %u\n", uMsr), rc);
1314
1315 /* Now that we're swapping MSRs during the world-switch, allow the guest to read/write them without causing VM-exits. */
1316 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1317 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
1318
1319 fAdded = true;
1320 }
1321
1322 /* Update the MSR values in the auto-load/store MSR area. */
1323 pGuestMsr->u32Msr = uMsr;
1324 pGuestMsr->u64Value = uGuestMsrValue;
1325
1326 /* Create/update the MSR slot in the host MSR area. */
1327 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1328 pHostMsr += i;
1329 pHostMsr->u32Msr = uMsr;
1330
1331 /*
1332 * Update the host MSR only when requested by the caller AND when we're
1333 * adding it to the auto-load/store area. Otherwise, it would have been
1334 * updated by hmR0VmxExportHostMsrs(). We do this for performance reasons.
1335 */
1336 bool fUpdatedMsrValue = false;
1337 if ( fAdded
1338 && fUpdateHostMsr)
1339 {
1340 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1341 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1342 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1343 fUpdatedMsrValue = true;
1344 }
1345
1346 if (pfAddedAndUpdated)
1347 *pfAddedAndUpdated = fUpdatedMsrValue;
1348 return VINF_SUCCESS;
1349}
1350
1351
1352/**
1353 * Removes a guest/host MSR pair to be swapped during the world-switch from the
1354 * auto-load/store MSR area in the VMCS.
1355 *
1356 * @returns VBox status code.
1357 * @param pVCpu The cross context virtual CPU structure.
1358 * @param uMsr The MSR.
1359 */
1360static int hmR0VmxRemoveAutoLoadStoreMsr(PVMCPU pVCpu, uint32_t uMsr)
1361{
1362 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1363 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1364 for (uint32_t i = 0; i < cMsrs; i++)
1365 {
1366 /* Find the MSR. */
1367 if (pGuestMsr->u32Msr == uMsr)
1368 {
1369 /* If it's the last MSR, simply reduce the count. */
1370 if (i == cMsrs - 1)
1371 {
1372 --cMsrs;
1373 break;
1374 }
1375
1376 /* Remove it by swapping the last MSR in place of it, and reducing the count. */
1377 PVMXAUTOMSR pLastGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1378 pLastGuestMsr += cMsrs - 1;
1379 pGuestMsr->u32Msr = pLastGuestMsr->u32Msr;
1380 pGuestMsr->u64Value = pLastGuestMsr->u64Value;
1381
1382 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1383 PVMXAUTOMSR pLastHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1384 pLastHostMsr += cMsrs - 1;
1385 pHostMsr->u32Msr = pLastHostMsr->u32Msr;
1386 pHostMsr->u64Value = pLastHostMsr->u64Value;
1387 --cMsrs;
1388 break;
1389 }
1390 pGuestMsr++;
1391 }
1392
1393 /* Update the VMCS if the count changed (meaning the MSR was found). */
1394 if (cMsrs != pVCpu->hm.s.vmx.cMsrs)
1395 {
1396 int rc = hmR0VmxSetAutoLoadStoreMsrCount(pVCpu, cMsrs);
1397 AssertRCReturn(rc, rc);
1398
1399 /* We're no longer swapping MSRs during the world-switch, intercept guest read/writes to them. */
1400 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1401 hmR0VmxSetMsrPermission(pVCpu, uMsr, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
1402
1403 Log4Func(("Removed MSR %#RX32 new cMsrs=%u\n", uMsr, pVCpu->hm.s.vmx.cMsrs));
1404 return VINF_SUCCESS;
1405 }
1406
1407 return VERR_NOT_FOUND;
1408}
1409
1410
1411/**
1412 * Checks if the specified guest MSR is part of the auto-load/store area in
1413 * the VMCS.
1414 *
1415 * @returns true if found, false otherwise.
1416 * @param pVCpu The cross context virtual CPU structure.
1417 * @param uMsr The MSR to find.
1418 */
1419static bool hmR0VmxIsAutoLoadStoreGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1420{
1421 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1422 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1423
1424 for (uint32_t i = 0; i < cMsrs; i++, pGuestMsr++)
1425 {
1426 if (pGuestMsr->u32Msr == uMsr)
1427 return true;
1428 }
1429 return false;
1430}
1431
1432
1433/**
1434 * Updates the value of all host MSRs in the auto-load/store area in the VMCS.
1435 *
1436 * @param pVCpu The cross context virtual CPU structure.
1437 *
1438 * @remarks No-long-jump zone!!!
1439 */
1440static void hmR0VmxUpdateAutoLoadStoreHostMsrs(PVMCPU pVCpu)
1441{
1442 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1443 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1444 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1445 uint32_t cMsrs = pVCpu->hm.s.vmx.cMsrs;
1446
1447 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1448 {
1449 AssertReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr);
1450
1451 /*
1452 * Performance hack for the host EFER MSR. We use the cached value rather than re-read it.
1453 * Strict builds will catch mismatches in hmR0VmxCheckAutoLoadStoreMsrs(). See @bugref{7368}.
1454 */
1455 if (pHostMsr->u32Msr == MSR_K6_EFER)
1456 pHostMsr->u64Value = pVCpu->CTX_SUFF(pVM)->hm.s.vmx.u64HostEfer;
1457 else
1458 pHostMsr->u64Value = ASMRdMsr(pHostMsr->u32Msr);
1459 }
1460
1461 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
1462}
1463
1464
1465/**
1466 * Saves a set of host MSRs to allow read/write passthru access to the guest and
1467 * perform lazy restoration of the host MSRs while leaving VT-x.
1468 *
1469 * @param pVCpu The cross context virtual CPU structure.
1470 *
1471 * @remarks No-long-jump zone!!!
1472 */
1473static void hmR0VmxLazySaveHostMsrs(PVMCPU pVCpu)
1474{
1475 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1476
1477 /*
1478 * Note: If you're adding MSRs here, make sure to update the MSR-bitmap permissions in hmR0VmxSetupProcCtls().
1479 */
1480 if (!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST))
1481 {
1482 Assert(!(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)); /* Guest MSRs better not be loaded now. */
1483#if HC_ARCH_BITS == 64
1484 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1485 {
1486 pVCpu->hm.s.vmx.u64HostLStarMsr = ASMRdMsr(MSR_K8_LSTAR);
1487 pVCpu->hm.s.vmx.u64HostStarMsr = ASMRdMsr(MSR_K6_STAR);
1488 pVCpu->hm.s.vmx.u64HostSFMaskMsr = ASMRdMsr(MSR_K8_SF_MASK);
1489 pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
1490 }
1491#endif
1492 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_SAVED_HOST;
1493 }
1494}
1495
1496
1497/**
1498 * Checks whether the MSR belongs to the set of guest MSRs that we restore
1499 * lazily while leaving VT-x.
1500 *
1501 * @returns true if it does, false otherwise.
1502 * @param pVCpu The cross context virtual CPU structure.
1503 * @param uMsr The MSR to check.
1504 */
1505static bool hmR0VmxIsLazyGuestMsr(PVMCPU pVCpu, uint32_t uMsr)
1506{
1507 NOREF(pVCpu);
1508#if HC_ARCH_BITS == 64
1509 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1510 {
1511 switch (uMsr)
1512 {
1513 case MSR_K8_LSTAR:
1514 case MSR_K6_STAR:
1515 case MSR_K8_SF_MASK:
1516 case MSR_K8_KERNEL_GS_BASE:
1517 return true;
1518 }
1519 }
1520#else
1521 RT_NOREF(pVCpu, uMsr);
1522#endif
1523 return false;
1524}
1525
1526
1527/**
1528 * Loads a set of guests MSRs to allow read/passthru to the guest.
1529 *
1530 * The name of this function is slightly confusing. This function does NOT
1531 * postpone loading, but loads the MSR right now. "hmR0VmxLazy" is simply a
1532 * common prefix for functions dealing with "lazy restoration" of the shared
1533 * MSRs.
1534 *
1535 * @param pVCpu The cross context virtual CPU structure.
1536 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
1537 * out-of-sync. Make sure to update the required fields
1538 * before using them.
1539 *
1540 * @remarks No-long-jump zone!!!
1541 */
1542static void hmR0VmxLazyLoadGuestMsrs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
1543{
1544 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1545 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1546
1547 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1548#if HC_ARCH_BITS == 64
1549 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1550 {
1551 /*
1552 * If the guest MSRs are not loaded -and- if all the guest MSRs are identical
1553 * to the MSRs on the CPU (which are the saved host MSRs, see assertion above) then
1554 * we can skip a few MSR writes.
1555 *
1556 * Otherwise, it implies either 1. they're not loaded, or 2. they're loaded but the
1557 * guest MSR values in the guest-CPU context might be different to what's currently
1558 * loaded in the CPU. In either case, we need to write the new guest MSR values to the
1559 * CPU, see @bugref{8728}.
1560 */
1561 if ( !(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1562 && pMixedCtx->msrKERNELGSBASE == pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr
1563 && pMixedCtx->msrLSTAR == pVCpu->hm.s.vmx.u64HostLStarMsr
1564 && pMixedCtx->msrSTAR == pVCpu->hm.s.vmx.u64HostStarMsr
1565 && pMixedCtx->msrSFMASK == pVCpu->hm.s.vmx.u64HostSFMaskMsr)
1566 {
1567#ifdef VBOX_STRICT
1568 Assert(ASMRdMsr(MSR_K8_KERNEL_GS_BASE) == pMixedCtx->msrKERNELGSBASE);
1569 Assert(ASMRdMsr(MSR_K8_LSTAR) == pMixedCtx->msrLSTAR);
1570 Assert(ASMRdMsr(MSR_K6_STAR) == pMixedCtx->msrSTAR);
1571 Assert(ASMRdMsr(MSR_K8_SF_MASK) == pMixedCtx->msrSFMASK);
1572#endif
1573 }
1574 else
1575 {
1576 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE);
1577 ASMWrMsr(MSR_K8_LSTAR, pMixedCtx->msrLSTAR);
1578 ASMWrMsr(MSR_K6_STAR, pMixedCtx->msrSTAR);
1579 ASMWrMsr(MSR_K8_SF_MASK, pMixedCtx->msrSFMASK);
1580 }
1581 }
1582#else
1583 RT_NOREF(pMixedCtx);
1584#endif
1585 pVCpu->hm.s.vmx.fLazyMsrs |= VMX_LAZY_MSRS_LOADED_GUEST;
1586}
1587
1588
1589/**
1590 * Performs lazy restoration of the set of host MSRs if they were previously
1591 * loaded with guest MSR values.
1592 *
1593 * @param pVCpu The cross context virtual CPU structure.
1594 *
1595 * @remarks No-long-jump zone!!!
1596 * @remarks The guest MSRs should have been saved back into the guest-CPU
1597 * context by hmR0VmxImportGuestState()!!!
1598 */
1599static void hmR0VmxLazyRestoreHostMsrs(PVMCPU pVCpu)
1600{
1601 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1602 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
1603
1604 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
1605 {
1606 Assert(pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_SAVED_HOST);
1607#if HC_ARCH_BITS == 64
1608 if (pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests)
1609 {
1610 ASMWrMsr(MSR_K8_LSTAR, pVCpu->hm.s.vmx.u64HostLStarMsr);
1611 ASMWrMsr(MSR_K6_STAR, pVCpu->hm.s.vmx.u64HostStarMsr);
1612 ASMWrMsr(MSR_K8_SF_MASK, pVCpu->hm.s.vmx.u64HostSFMaskMsr);
1613 ASMWrMsr(MSR_K8_KERNEL_GS_BASE, pVCpu->hm.s.vmx.u64HostKernelGSBaseMsr);
1614 }
1615#endif
1616 }
1617 pVCpu->hm.s.vmx.fLazyMsrs &= ~(VMX_LAZY_MSRS_LOADED_GUEST | VMX_LAZY_MSRS_SAVED_HOST);
1618}
1619
1620
1621/**
1622 * Verifies that our cached values of the VMCS controls are all
1623 * consistent with what's actually present in the VMCS.
1624 *
1625 * @returns VBox status code.
1626 * @param pVCpu The cross context virtual CPU structure.
1627 */
1628static int hmR0VmxCheckVmcsCtls(PVMCPU pVCpu)
1629{
1630 uint32_t u32Val;
1631 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
1632 AssertRCReturn(rc, rc);
1633 AssertMsgReturn(pVCpu->hm.s.vmx.u32EntryCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32EntryCtls, u32Val),
1634 VERR_VMX_ENTRY_CTLS_CACHE_INVALID);
1635
1636 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val);
1637 AssertRCReturn(rc, rc);
1638 AssertMsgReturn(pVCpu->hm.s.vmx.u32ExitCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ExitCtls, u32Val),
1639 VERR_VMX_EXIT_CTLS_CACHE_INVALID);
1640
1641 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val);
1642 AssertRCReturn(rc, rc);
1643 AssertMsgReturn(pVCpu->hm.s.vmx.u32PinCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32PinCtls, u32Val),
1644 VERR_VMX_PIN_EXEC_CTLS_CACHE_INVALID);
1645
1646 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val);
1647 AssertRCReturn(rc, rc);
1648 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls == u32Val, ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls, u32Val),
1649 VERR_VMX_PROC_EXEC_CTLS_CACHE_INVALID);
1650
1651 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
1652 {
1653 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val);
1654 AssertRCReturn(rc, rc);
1655 AssertMsgReturn(pVCpu->hm.s.vmx.u32ProcCtls2 == u32Val,
1656 ("Cache=%#RX32 VMCS=%#RX32", pVCpu->hm.s.vmx.u32ProcCtls2, u32Val),
1657 VERR_VMX_PROC_EXEC2_CTLS_CACHE_INVALID);
1658 }
1659
1660 return VINF_SUCCESS;
1661}
1662
1663
1664#ifdef VBOX_STRICT
1665/**
1666 * Verifies that our cached host EFER value has not changed
1667 * since we cached it.
1668 *
1669 * @param pVCpu The cross context virtual CPU structure.
1670 */
1671static void hmR0VmxCheckHostEferMsr(PVMCPU pVCpu)
1672{
1673 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1674
1675 if (pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
1676 {
1677 uint64_t u64Val;
1678 int rc = VMXReadVmcs64(VMX_VMCS64_HOST_EFER_FULL, &u64Val);
1679 AssertRC(rc);
1680
1681 uint64_t u64HostEferMsr = ASMRdMsr(MSR_K6_EFER);
1682 AssertMsgReturnVoid(u64HostEferMsr == u64Val, ("u64HostEferMsr=%#RX64 u64Val=%#RX64\n", u64HostEferMsr, u64Val));
1683 }
1684}
1685
1686
1687/**
1688 * Verifies whether the guest/host MSR pairs in the auto-load/store area in the
1689 * VMCS are correct.
1690 *
1691 * @param pVCpu The cross context virtual CPU structure.
1692 */
1693static void hmR0VmxCheckAutoLoadStoreMsrs(PVMCPU pVCpu)
1694{
1695 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
1696
1697 /* Verify MSR counts in the VMCS are what we think it should be. */
1698 uint32_t cMsrs;
1699 int rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1700 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1701
1702 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &cMsrs); AssertRC(rc);
1703 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1704
1705 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &cMsrs); AssertRC(rc);
1706 Assert(cMsrs == pVCpu->hm.s.vmx.cMsrs);
1707
1708 PVMXAUTOMSR pHostMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvHostMsr;
1709 PVMXAUTOMSR pGuestMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
1710 for (uint32_t i = 0; i < cMsrs; i++, pHostMsr++, pGuestMsr++)
1711 {
1712 /* Verify that the MSRs are paired properly and that the host MSR has the correct value. */
1713 AssertMsgReturnVoid(pHostMsr->u32Msr == pGuestMsr->u32Msr, ("HostMsr=%#RX32 GuestMsr=%#RX32 cMsrs=%u\n", pHostMsr->u32Msr,
1714 pGuestMsr->u32Msr, cMsrs));
1715
1716 uint64_t u64Msr = ASMRdMsr(pHostMsr->u32Msr);
1717 AssertMsgReturnVoid(pHostMsr->u64Value == u64Msr, ("u32Msr=%#RX32 VMCS Value=%#RX64 ASMRdMsr=%#RX64 cMsrs=%u\n",
1718 pHostMsr->u32Msr, pHostMsr->u64Value, u64Msr, cMsrs));
1719
1720 /* Verify that the permissions are as expected in the MSR bitmap. */
1721 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
1722 {
1723 VMXMSREXITREAD enmRead;
1724 VMXMSREXITWRITE enmWrite;
1725 rc = hmR0VmxGetMsrPermission(pVCpu, pGuestMsr->u32Msr, &enmRead, &enmWrite);
1726 AssertMsgReturnVoid(rc == VINF_SUCCESS, ("hmR0VmxGetMsrPermission! failed. rc=%Rrc\n", rc));
1727 if (pGuestMsr->u32Msr == MSR_K6_EFER)
1728 {
1729 AssertMsgReturnVoid(enmRead == VMXMSREXIT_INTERCEPT_READ, ("Passthru read for EFER!?\n"));
1730 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_INTERCEPT_WRITE, ("Passthru write for EFER!?\n"));
1731 }
1732 else
1733 {
1734 AssertMsgReturnVoid(enmRead == VMXMSREXIT_PASSTHRU_READ, ("u32Msr=%#RX32 cMsrs=%u No passthru read!\n",
1735 pGuestMsr->u32Msr, cMsrs));
1736 AssertMsgReturnVoid(enmWrite == VMXMSREXIT_PASSTHRU_WRITE, ("u32Msr=%#RX32 cMsrs=%u No passthru write!\n",
1737 pGuestMsr->u32Msr, cMsrs));
1738 }
1739 }
1740 }
1741}
1742#endif /* VBOX_STRICT */
1743
1744
1745/**
1746 * Flushes the TLB using EPT.
1747 *
1748 * @returns VBox status code.
1749 * @param pVCpu The cross context virtual CPU structure of the calling
1750 * EMT. Can be NULL depending on @a enmFlush.
1751 * @param enmFlush Type of flush.
1752 *
1753 * @remarks Caller is responsible for making sure this function is called only
1754 * when NestedPaging is supported and providing @a enmFlush that is
1755 * supported by the CPU.
1756 * @remarks Can be called with interrupts disabled.
1757 */
1758static void hmR0VmxFlushEpt(PVMCPU pVCpu, VMXFLUSHEPT enmFlush)
1759{
1760 uint64_t au64Descriptor[2];
1761 if (enmFlush == VMXFLUSHEPT_ALL_CONTEXTS)
1762 au64Descriptor[0] = 0;
1763 else
1764 {
1765 Assert(pVCpu);
1766 au64Descriptor[0] = pVCpu->hm.s.vmx.HCPhysEPTP;
1767 }
1768 au64Descriptor[1] = 0; /* MBZ. Intel spec. 33.3 "VMX Instructions" */
1769
1770 int rc = VMXR0InvEPT(enmFlush, &au64Descriptor[0]);
1771 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvEPT %#x %RGv failed with %Rrc\n", enmFlush, pVCpu ? pVCpu->hm.s.vmx.HCPhysEPTP : 0,
1772 rc));
1773 if ( RT_SUCCESS(rc)
1774 && pVCpu)
1775 {
1776 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushNestedPaging);
1777 }
1778}
1779
1780
1781/**
1782 * Flushes the TLB using VPID.
1783 *
1784 * @returns VBox status code.
1785 * @param pVCpu The cross context virtual CPU structure of the calling
1786 * EMT. Can be NULL depending on @a enmFlush.
1787 * @param enmFlush Type of flush.
1788 * @param GCPtr Virtual address of the page to flush (can be 0 depending
1789 * on @a enmFlush).
1790 *
1791 * @remarks Can be called with interrupts disabled.
1792 */
1793static void hmR0VmxFlushVpid(PVMCPU pVCpu, VMXFLUSHVPID enmFlush, RTGCPTR GCPtr)
1794{
1795 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid);
1796
1797 uint64_t au64Descriptor[2];
1798 if (enmFlush == VMXFLUSHVPID_ALL_CONTEXTS)
1799 {
1800 au64Descriptor[0] = 0;
1801 au64Descriptor[1] = 0;
1802 }
1803 else
1804 {
1805 AssertPtr(pVCpu);
1806 AssertMsg(pVCpu->hm.s.uCurrentAsid != 0, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1807 AssertMsg(pVCpu->hm.s.uCurrentAsid <= UINT16_MAX, ("VMXR0InvVPID: invalid ASID %lu\n", pVCpu->hm.s.uCurrentAsid));
1808 au64Descriptor[0] = pVCpu->hm.s.uCurrentAsid;
1809 au64Descriptor[1] = GCPtr;
1810 }
1811
1812 int rc = VMXR0InvVPID(enmFlush, &au64Descriptor[0]);
1813 AssertMsg(rc == VINF_SUCCESS, ("VMXR0InvVPID %#x %u %RGv failed with %Rrc\n", enmFlush,
1814 pVCpu ? pVCpu->hm.s.uCurrentAsid : 0, GCPtr, rc));
1815 if ( RT_SUCCESS(rc)
1816 && pVCpu)
1817 {
1818 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushAsid);
1819 }
1820 NOREF(rc);
1821}
1822
1823
1824/**
1825 * Invalidates a guest page by guest virtual address. Only relevant for
1826 * EPT/VPID, otherwise there is nothing really to invalidate.
1827 *
1828 * @returns VBox status code.
1829 * @param pVCpu The cross context virtual CPU structure.
1830 * @param GCVirt Guest virtual address of the page to invalidate.
1831 */
1832VMMR0DECL(int) VMXR0InvalidatePage(PVMCPU pVCpu, RTGCPTR GCVirt)
1833{
1834 AssertPtr(pVCpu);
1835 LogFlowFunc(("pVCpu=%p GCVirt=%RGv\n", pVCpu, GCVirt));
1836
1837 bool fFlushPending = VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_TLB_FLUSH);
1838 if (!fFlushPending)
1839 {
1840 /*
1841 * We must invalidate the guest TLB entry in either case, we cannot ignore it even for
1842 * the EPT case. See @bugref{6043} and @bugref{6177}.
1843 *
1844 * Set the VMCPU_FF_TLB_FLUSH force flag and flush before VM-entry in hmR0VmxFlushTLB*()
1845 * as this function maybe called in a loop with individual addresses.
1846 */
1847 PVM pVM = pVCpu->CTX_SUFF(pVM);
1848 if (pVM->hm.s.vmx.fVpid)
1849 {
1850 bool fVpidFlush = RT_BOOL(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR);
1851
1852#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
1853 /*
1854 * Workaround Erratum BV75, AAJ159 and others that affect several Intel CPUs
1855 * where executing INVVPID outside 64-bit mode does not flush translations of
1856 * 64-bit linear addresses, see @bugref{6208#c72}.
1857 */
1858 if (RT_HI_U32(GCVirt))
1859 fVpidFlush = false;
1860#endif
1861
1862 if (fVpidFlush)
1863 {
1864 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_INDIV_ADDR, GCVirt);
1865 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbInvlpgVirt);
1866 }
1867 else
1868 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1869 }
1870 else if (pVM->hm.s.fNestedPaging)
1871 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
1872 }
1873
1874 return VINF_SUCCESS;
1875}
1876
1877
1878/**
1879 * Dummy placeholder for tagged-TLB flush handling before VM-entry. Used in the
1880 * case where neither EPT nor VPID is supported by the CPU.
1881 *
1882 * @param pVCpu The cross context virtual CPU structure.
1883 * @param pCpu Pointer to the global HM struct.
1884 *
1885 * @remarks Called with interrupts disabled.
1886 */
1887static void hmR0VmxFlushTaggedTlbNone(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1888{
1889 AssertPtr(pVCpu);
1890 AssertPtr(pCpu);
1891
1892 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH);
1893
1894 Assert(pCpu->idCpu != NIL_RTCPUID);
1895 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1896 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1897 pVCpu->hm.s.fForceTLBFlush = false;
1898 return;
1899}
1900
1901
1902/**
1903 * Flushes the tagged-TLB entries for EPT+VPID CPUs as necessary.
1904 *
1905 * @param pVCpu The cross context virtual CPU structure.
1906 * @param pCpu Pointer to the global HM CPU struct.
1907 *
1908 * @remarks All references to "ASID" in this function pertains to "VPID" in Intel's
1909 * nomenclature. The reason is, to avoid confusion in compare statements
1910 * since the host-CPU copies are named "ASID".
1911 *
1912 * @remarks Called with interrupts disabled.
1913 */
1914static void hmR0VmxFlushTaggedTlbBoth(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
1915{
1916#ifdef VBOX_WITH_STATISTICS
1917 bool fTlbFlushed = false;
1918# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { fTlbFlushed = true; } while (0)
1919# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { \
1920 if (!fTlbFlushed) \
1921 STAM_COUNTER_INC(&pVCpu->hm.s.StatNoFlushTlbWorldSwitch); \
1922 } while (0)
1923#else
1924# define HMVMX_SET_TAGGED_TLB_FLUSHED() do { } while (0)
1925# define HMVMX_UPDATE_FLUSH_SKIPPED_STAT() do { } while (0)
1926#endif
1927
1928 AssertPtr(pCpu);
1929 AssertPtr(pVCpu);
1930 Assert(pCpu->idCpu != NIL_RTCPUID);
1931
1932 PVM pVM = pVCpu->CTX_SUFF(pVM);
1933 AssertMsg(pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid,
1934 ("hmR0VmxFlushTaggedTlbBoth cannot be invoked unless NestedPaging & VPID are enabled."
1935 "fNestedPaging=%RTbool fVpid=%RTbool", pVM->hm.s.fNestedPaging, pVM->hm.s.vmx.fVpid));
1936
1937 /*
1938 * Force a TLB flush for the first world-switch if the current CPU differs from the one we
1939 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
1940 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
1941 * cannot reuse the current ASID anymore.
1942 */
1943 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
1944 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
1945 {
1946 ++pCpu->uCurrentAsid;
1947 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
1948 {
1949 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0. */
1950 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
1951 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
1952 }
1953
1954 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
1955 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
1956 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
1957
1958 /*
1959 * Flush by EPT when we get rescheduled to a new host CPU to ensure EPT-only tagged mappings are also
1960 * invalidated. We don't need to flush-by-VPID here as flushing by EPT covers it. See @bugref{6568}.
1961 */
1962 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1963 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
1964 HMVMX_SET_TAGGED_TLB_FLUSHED();
1965 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH); /* Already flushed-by-EPT, skip doing it again below. */
1966 }
1967
1968 /* Check for explicit TLB flushes. */
1969 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
1970 {
1971 /*
1972 * Changes to the EPT paging structure by VMM requires flushing-by-EPT as the CPU
1973 * creates guest-physical (ie. only EPT-tagged) mappings while traversing the EPT
1974 * tables when EPT is in use. Flushing-by-VPID will only flush linear (only
1975 * VPID-tagged) and combined (EPT+VPID tagged) mappings but not guest-physical
1976 * mappings, see @bugref{6568}.
1977 *
1978 * See Intel spec. 28.3.2 "Creating and Using Cached Translation Information".
1979 */
1980 hmR0VmxFlushEpt(pVCpu, pVM->hm.s.vmx.enmFlushEpt);
1981 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
1982 HMVMX_SET_TAGGED_TLB_FLUSHED();
1983 }
1984
1985 pVCpu->hm.s.fForceTLBFlush = false;
1986 HMVMX_UPDATE_FLUSH_SKIPPED_STAT();
1987
1988 Assert(pVCpu->hm.s.idLastCpu == pCpu->idCpu);
1989 Assert(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes);
1990 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
1991 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
1992 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
1993 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
1994 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
1995 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
1996 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
1997
1998 /* Update VMCS with the VPID. */
1999 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2000 AssertRC(rc);
2001
2002#undef HMVMX_SET_TAGGED_TLB_FLUSHED
2003}
2004
2005
2006/**
2007 * Flushes the tagged-TLB entries for EPT CPUs as necessary.
2008 *
2009 * @returns VBox status code.
2010 * @param pVCpu The cross context virtual CPU structure.
2011 * @param pCpu Pointer to the global HM CPU struct.
2012 *
2013 * @remarks Called with interrupts disabled.
2014 */
2015static void hmR0VmxFlushTaggedTlbEpt(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2016{
2017 AssertPtr(pVCpu);
2018 AssertPtr(pCpu);
2019 Assert(pCpu->idCpu != NIL_RTCPUID);
2020 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked without NestedPaging."));
2021 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTaggedTlbEpt cannot be invoked with VPID."));
2022
2023 /*
2024 * Force a TLB flush for the first world-switch if the current CPU differs from the one we ran on last.
2025 * A change in the TLB flush count implies the host CPU is online after a suspend/resume.
2026 */
2027 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2028 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2029 {
2030 pVCpu->hm.s.fForceTLBFlush = true;
2031 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2032 }
2033
2034 /* Check for explicit TLB flushes. */
2035 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2036 {
2037 pVCpu->hm.s.fForceTLBFlush = true;
2038 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2039 }
2040
2041 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2042 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2043
2044 if (pVCpu->hm.s.fForceTLBFlush)
2045 {
2046 hmR0VmxFlushEpt(pVCpu, pVCpu->CTX_SUFF(pVM)->hm.s.vmx.enmFlushEpt);
2047 pVCpu->hm.s.fForceTLBFlush = false;
2048 }
2049}
2050
2051
2052/**
2053 * Flushes the tagged-TLB entries for VPID CPUs as necessary.
2054 *
2055 * @returns VBox status code.
2056 * @param pVCpu The cross context virtual CPU structure.
2057 * @param pCpu Pointer to the global HM CPU struct.
2058 *
2059 * @remarks Called with interrupts disabled.
2060 */
2061static void hmR0VmxFlushTaggedTlbVpid(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2062{
2063 AssertPtr(pVCpu);
2064 AssertPtr(pCpu);
2065 Assert(pCpu->idCpu != NIL_RTCPUID);
2066 AssertMsg(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid, ("hmR0VmxFlushTlbVpid cannot be invoked without VPID."));
2067 AssertMsg(!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging, ("hmR0VmxFlushTlbVpid cannot be invoked with NestedPaging"));
2068
2069 /*
2070 * Force a TLB flush for the first world switch if the current CPU differs from the one we
2071 * ran on last. If the TLB flush count changed, another VM (VCPU rather) has hit the ASID
2072 * limit while flushing the TLB or the host CPU is online after a suspend/resume, so we
2073 * cannot reuse the current ASID anymore.
2074 */
2075 if ( pVCpu->hm.s.idLastCpu != pCpu->idCpu
2076 || pVCpu->hm.s.cTlbFlushes != pCpu->cTlbFlushes)
2077 {
2078 pVCpu->hm.s.fForceTLBFlush = true;
2079 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlbWorldSwitch);
2080 }
2081
2082 /* Check for explicit TLB flushes. */
2083 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_TLB_FLUSH))
2084 {
2085 /*
2086 * If we ever support VPID flush combinations other than ALL or SINGLE-context (see
2087 * hmR0VmxSetupTaggedTlb()) we would need to explicitly flush in this case (add an
2088 * fExplicitFlush = true here and change the pCpu->fFlushAsidBeforeUse check below to
2089 * include fExplicitFlush's too) - an obscure corner case.
2090 */
2091 pVCpu->hm.s.fForceTLBFlush = true;
2092 STAM_COUNTER_INC(&pVCpu->hm.s.StatFlushTlb);
2093 }
2094
2095 PVM pVM = pVCpu->CTX_SUFF(pVM);
2096 pVCpu->hm.s.idLastCpu = pCpu->idCpu;
2097 if (pVCpu->hm.s.fForceTLBFlush)
2098 {
2099 ++pCpu->uCurrentAsid;
2100 if (pCpu->uCurrentAsid >= pVM->hm.s.uMaxAsid)
2101 {
2102 pCpu->uCurrentAsid = 1; /* Wraparound to 1; host uses 0 */
2103 pCpu->cTlbFlushes++; /* All VCPUs that run on this host CPU must use a new VPID. */
2104 pCpu->fFlushAsidBeforeUse = true; /* All VCPUs that run on this host CPU must flush their new VPID before use. */
2105 }
2106
2107 pVCpu->hm.s.fForceTLBFlush = false;
2108 pVCpu->hm.s.cTlbFlushes = pCpu->cTlbFlushes;
2109 pVCpu->hm.s.uCurrentAsid = pCpu->uCurrentAsid;
2110 if (pCpu->fFlushAsidBeforeUse)
2111 {
2112 if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_SINGLE_CONTEXT)
2113 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_SINGLE_CONTEXT, 0 /* GCPtr */);
2114 else if (pVM->hm.s.vmx.enmFlushVpid == VMXFLUSHVPID_ALL_CONTEXTS)
2115 {
2116 hmR0VmxFlushVpid(pVCpu, VMXFLUSHVPID_ALL_CONTEXTS, 0 /* GCPtr */);
2117 pCpu->fFlushAsidBeforeUse = false;
2118 }
2119 else
2120 {
2121 /* hmR0VmxSetupTaggedTlb() ensures we never get here. Paranoia. */
2122 AssertMsgFailed(("Unsupported VPID-flush context type.\n"));
2123 }
2124 }
2125 }
2126
2127 AssertMsg(pVCpu->hm.s.cTlbFlushes == pCpu->cTlbFlushes,
2128 ("Flush count mismatch for cpu %d (%u vs %u)\n", pCpu->idCpu, pVCpu->hm.s.cTlbFlushes, pCpu->cTlbFlushes));
2129 AssertMsg(pCpu->uCurrentAsid >= 1 && pCpu->uCurrentAsid < pVM->hm.s.uMaxAsid,
2130 ("Cpu[%u] uCurrentAsid=%u cTlbFlushes=%u pVCpu->idLastCpu=%u pVCpu->cTlbFlushes=%u\n", pCpu->idCpu,
2131 pCpu->uCurrentAsid, pCpu->cTlbFlushes, pVCpu->hm.s.idLastCpu, pVCpu->hm.s.cTlbFlushes));
2132 AssertMsg(pVCpu->hm.s.uCurrentAsid >= 1 && pVCpu->hm.s.uCurrentAsid < pVM->hm.s.uMaxAsid,
2133 ("Cpu[%u] pVCpu->uCurrentAsid=%u\n", pCpu->idCpu, pVCpu->hm.s.uCurrentAsid));
2134
2135 int rc = VMXWriteVmcs32(VMX_VMCS16_VPID, pVCpu->hm.s.uCurrentAsid);
2136 AssertRC(rc);
2137}
2138
2139
2140/**
2141 * Flushes the guest TLB entry based on CPU capabilities.
2142 *
2143 * @param pVCpu The cross context virtual CPU structure.
2144 * @param pCpu Pointer to the global HM CPU struct.
2145 */
2146DECLINLINE(void) hmR0VmxFlushTaggedTlb(PVMCPU pVCpu, PHMGLOBALCPUINFO pCpu)
2147{
2148#ifdef HMVMX_ALWAYS_FLUSH_TLB
2149 VMCPU_FF_SET(pVCpu, VMCPU_FF_TLB_FLUSH);
2150#endif
2151 PVM pVM = pVCpu->CTX_SUFF(pVM);
2152 switch (pVM->hm.s.vmx.uFlushTaggedTlb)
2153 {
2154 case HMVMX_FLUSH_TAGGED_TLB_EPT_VPID: hmR0VmxFlushTaggedTlbBoth(pVCpu, pCpu); break;
2155 case HMVMX_FLUSH_TAGGED_TLB_EPT: hmR0VmxFlushTaggedTlbEpt(pVCpu, pCpu); break;
2156 case HMVMX_FLUSH_TAGGED_TLB_VPID: hmR0VmxFlushTaggedTlbVpid(pVCpu, pCpu); break;
2157 case HMVMX_FLUSH_TAGGED_TLB_NONE: hmR0VmxFlushTaggedTlbNone(pVCpu, pCpu); break;
2158 default:
2159 AssertMsgFailed(("Invalid flush-tag function identifier\n"));
2160 break;
2161 }
2162 /* Don't assert that VMCPU_FF_TLB_FLUSH should no longer be pending. It can be set by other EMTs. */
2163}
2164
2165
2166/**
2167 * Sets up the appropriate tagged TLB-flush level and handler for flushing guest
2168 * TLB entries from the host TLB before VM-entry.
2169 *
2170 * @returns VBox status code.
2171 * @param pVM The cross context VM structure.
2172 */
2173static int hmR0VmxSetupTaggedTlb(PVM pVM)
2174{
2175 /*
2176 * Determine optimal flush type for Nested Paging.
2177 * We cannot ignore EPT if no suitable flush-types is supported by the CPU as we've already setup unrestricted
2178 * guest execution (see hmR3InitFinalizeR0()).
2179 */
2180 if (pVM->hm.s.fNestedPaging)
2181 {
2182 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT)
2183 {
2184 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_SINGLE_CONTEXT)
2185 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_SINGLE_CONTEXT;
2186 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVEPT_ALL_CONTEXTS)
2187 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_ALL_CONTEXTS;
2188 else
2189 {
2190 /* Shouldn't happen. EPT is supported but no suitable flush-types supported. */
2191 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2192 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_FLUSH_TYPE_UNSUPPORTED;
2193 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2194 }
2195
2196 /* Make sure the write-back cacheable memory type for EPT is supported. */
2197 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EMT_WB)))
2198 {
2199 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2200 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_MEM_TYPE_NOT_WB;
2201 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2202 }
2203
2204 /* EPT requires a page-walk length of 4. */
2205 if (RT_UNLIKELY(!(pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_PAGE_WALK_LENGTH_4)))
2206 {
2207 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2208 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_PAGE_WALK_LENGTH_UNSUPPORTED;
2209 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2210 }
2211 }
2212 else
2213 {
2214 /* Shouldn't happen. EPT is supported but INVEPT instruction is not supported. */
2215 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NOT_SUPPORTED;
2216 pVM->aCpus[0].hm.s.u32HMError = VMX_UFC_EPT_INVEPT_UNAVAILABLE;
2217 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2218 }
2219 }
2220
2221 /*
2222 * Determine optimal flush type for VPID.
2223 */
2224 if (pVM->hm.s.vmx.fVpid)
2225 {
2226 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID)
2227 {
2228 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT)
2229 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_SINGLE_CONTEXT;
2230 else if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_ALL_CONTEXTS)
2231 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_ALL_CONTEXTS;
2232 else
2233 {
2234 /* Neither SINGLE nor ALL-context flush types for VPID is supported by the CPU. Ignore VPID capability. */
2235 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_INDIV_ADDR)
2236 LogRelFunc(("Only INDIV_ADDR supported. Ignoring VPID.\n"));
2237 if (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_INVVPID_SINGLE_CONTEXT_RETAIN_GLOBALS)
2238 LogRelFunc(("Only SINGLE_CONTEXT_RETAIN_GLOBALS supported. Ignoring VPID.\n"));
2239 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2240 pVM->hm.s.vmx.fVpid = false;
2241 }
2242 }
2243 else
2244 {
2245 /* Shouldn't happen. VPID is supported but INVVPID is not supported by the CPU. Ignore VPID capability. */
2246 Log4Func(("VPID supported without INVEPT support. Ignoring VPID.\n"));
2247 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NOT_SUPPORTED;
2248 pVM->hm.s.vmx.fVpid = false;
2249 }
2250 }
2251
2252 /*
2253 * Setup the handler for flushing tagged-TLBs.
2254 */
2255 if (pVM->hm.s.fNestedPaging && pVM->hm.s.vmx.fVpid)
2256 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT_VPID;
2257 else if (pVM->hm.s.fNestedPaging)
2258 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_EPT;
2259 else if (pVM->hm.s.vmx.fVpid)
2260 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_VPID;
2261 else
2262 pVM->hm.s.vmx.uFlushTaggedTlb = HMVMX_FLUSH_TAGGED_TLB_NONE;
2263 return VINF_SUCCESS;
2264}
2265
2266
2267/**
2268 * Sets up pin-based VM-execution controls in the VMCS.
2269 *
2270 * @returns VBox status code.
2271 * @param pVCpu The cross context virtual CPU structure.
2272 *
2273 * @remarks We don't really care about optimizing vmwrites here as it's done only
2274 * once per VM and hence we don't care about VMCS-field cache comparisons.
2275 */
2276static int hmR0VmxSetupPinCtls(PVMCPU pVCpu)
2277{
2278 PVM pVM = pVCpu->CTX_SUFF(pVM);
2279 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0; /* Bits set here must always be set. */
2280 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1; /* Bits cleared here must always be cleared. */
2281
2282 fVal |= VMX_VMCS_CTRL_PIN_EXEC_EXT_INT_EXIT /* External interrupts cause a VM-exit. */
2283 | VMX_VMCS_CTRL_PIN_EXEC_NMI_EXIT; /* Non-maskable interrupts (NMIs) cause a VM-exit. */
2284
2285 if (pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
2286 fVal |= VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI; /* Use virtual NMIs and virtual-NMI blocking features. */
2287
2288 /* Enable the VMX preemption timer. */
2289 if (pVM->hm.s.vmx.fUsePreemptTimer)
2290 {
2291 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER);
2292 fVal |= VMX_VMCS_CTRL_PIN_EXEC_PREEMPT_TIMER;
2293 }
2294
2295#if 0
2296 /* Enable posted-interrupt processing. */
2297 if (pVM->hm.s.fPostedIntrs)
2298 {
2299 Assert(pVM->hm.s.vmx.Msrs.VmxPinCtls.n.allowed1 & VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR);
2300 Assert(pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT);
2301 fVal |= VMX_VMCS_CTRL_PIN_EXEC_POSTED_INTR;
2302 }
2303#endif
2304
2305 if ((fVal & fZap) != fVal)
2306 {
2307 LogRelFunc(("Invalid pin-based VM-execution controls combo! Cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2308 pVM->hm.s.vmx.Msrs.VmxPinCtls.n.disallowed0, fVal, fZap));
2309 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PIN_EXEC;
2310 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2311 }
2312
2313 /* Commit it to the VMCS and update our cache. */
2314 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, fVal);
2315 AssertRCReturn(rc, rc);
2316 pVCpu->hm.s.vmx.u32PinCtls = fVal;
2317
2318 return VINF_SUCCESS;
2319}
2320
2321
2322/**
2323 * Sets up secondary processor-based VM-execution controls in the VMCS.
2324 *
2325 * @returns VBox status code.
2326 * @param pVCpu The cross context virtual CPU structure.
2327 *
2328 * @remarks We don't really care about optimizing vmwrites here as it's done only
2329 * once per VM and hence we don't care about VMCS-field cache comparisons.
2330 */
2331static int hmR0VmxSetupProcCtls2(PVMCPU pVCpu)
2332{
2333 PVM pVM = pVCpu->CTX_SUFF(pVM);
2334 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0; /* Bits set here must be set in the VMCS. */
2335 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2336
2337 /* WBINVD causes a VM-exit. */
2338 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT)
2339 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT;
2340
2341 /* Enable EPT (aka nested-paging). */
2342 if (pVM->hm.s.fNestedPaging)
2343 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_EPT;
2344
2345 /*
2346 * Enable the INVPCID instruction if supported by the hardware and we expose
2347 * it to the guest. Without this, guest executing INVPCID would cause a #UD.
2348 */
2349 if ( (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_INVPCID)
2350 && pVM->cpum.ro.GuestFeatures.fInvpcid)
2351 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_INVPCID;
2352
2353 /* Enable VPID. */
2354 if (pVM->hm.s.vmx.fVpid)
2355 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VPID;
2356
2357 /* Enable Unrestricted guest execution. */
2358 if (pVM->hm.s.vmx.fUnrestrictedGuest)
2359 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_UNRESTRICTED_GUEST;
2360
2361#if 0
2362 if (pVM->hm.s.fVirtApicRegs)
2363 {
2364 /* Enable APIC-register virtualization. */
2365 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT);
2366 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_APIC_REG_VIRT;
2367
2368 /* Enable virtual-interrupt delivery. */
2369 Assert(pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY);
2370 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_INTR_DELIVERY;
2371 }
2372#endif
2373
2374 /* Enable Virtual-APIC page accesses if supported by the CPU. This is where the TPR shadow resides. */
2375 /** @todo VIRT_X2APIC support, it's mutually exclusive with this. So must be
2376 * done dynamically. */
2377 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
2378 {
2379 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
2380 Assert(!(pVM->hm.s.vmx.HCPhysApicAccess & 0xfff)); /* Bits 11:0 MBZ. */
2381 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC; /* Virtualize APIC accesses. */
2382 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL, pVM->hm.s.vmx.HCPhysApicAccess);
2383 AssertRCReturn(rc, rc);
2384 }
2385
2386 /* Enable RDTSCP. */
2387 if (pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
2388 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP;
2389
2390 /* Enable Pause-Loop exiting. */
2391 if ( pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT
2392 && pVM->hm.s.vmx.cPleGapTicks
2393 && pVM->hm.s.vmx.cPleWindowTicks)
2394 {
2395 fVal |= VMX_VMCS_CTRL_PROC_EXEC2_PAUSE_LOOP_EXIT;
2396
2397 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_GAP, pVM->hm.s.vmx.cPleGapTicks);
2398 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PLE_WINDOW, pVM->hm.s.vmx.cPleWindowTicks);
2399 AssertRCReturn(rc, rc);
2400 }
2401
2402 if ((fVal & fZap) != fVal)
2403 {
2404 LogRelFunc(("Invalid secondary processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2405 pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.disallowed0, fVal, fZap));
2406 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC2;
2407 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2408 }
2409
2410 /* Commit it to the VMCS and update our cache. */
2411 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, fVal);
2412 AssertRCReturn(rc, rc);
2413 pVCpu->hm.s.vmx.u32ProcCtls2 = fVal;
2414
2415 return VINF_SUCCESS;
2416}
2417
2418
2419/**
2420 * Sets up processor-based VM-execution controls in the VMCS.
2421 *
2422 * @returns VBox status code.
2423 * @param pVCpu The cross context virtual CPU structure.
2424 *
2425 * @remarks We don't really care about optimizing vmwrites here as it's done only
2426 * once per VM and hence we don't care about VMCS-field cache comparisons.
2427 */
2428static int hmR0VmxSetupProcCtls(PVMCPU pVCpu)
2429{
2430 PVM pVM = pVCpu->CTX_SUFF(pVM);
2431 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0; /* Bits set here must be set in the VMCS. */
2432 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
2433
2434 fVal |= VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT /* HLT causes a VM-exit. */
2435 | VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING /* Use TSC-offsetting. */
2436 | VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT /* MOV DRx causes a VM-exit. */
2437 | VMX_VMCS_CTRL_PROC_EXEC_UNCOND_IO_EXIT /* All IO instructions cause a VM-exit. */
2438 | VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT /* RDPMC causes a VM-exit. */
2439 | VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT /* MONITOR causes a VM-exit. */
2440 | VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT; /* MWAIT causes a VM-exit. */
2441
2442 /* We toggle VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT later, check if it's not -always- needed to be set or clear. */
2443 if ( !(pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT)
2444 || (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0 & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT))
2445 {
2446 LogRelFunc(("Unsupported VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT combo!"));
2447 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_MOV_DRX_EXIT;
2448 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2449 }
2450
2451 /* Without Nested Paging, INVLPG (also affects INVPCID) and MOV CR3 instructions should cause VM-exits. */
2452 if (!pVM->hm.s.fNestedPaging)
2453 {
2454 Assert(!pVM->hm.s.vmx.fUnrestrictedGuest); /* Paranoia. */
2455 fVal |= VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT
2456 | VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
2457 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
2458 }
2459
2460 /* Use TPR shadowing if supported by the CPU. */
2461 if ( PDMHasApic(pVM)
2462 && pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
2463 {
2464 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
2465 Assert(!(pVCpu->hm.s.vmx.HCPhysVirtApic & 0xfff)); /* Bits 11:0 MBZ. */
2466 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, 0);
2467 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL, pVCpu->hm.s.vmx.HCPhysVirtApic);
2468 AssertRCReturn(rc, rc);
2469
2470 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* CR8 reads from the Virtual-APIC page. */
2471 /* CR8 writes cause a VM-exit based on TPR threshold. */
2472 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT));
2473 Assert(!(fVal & VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT));
2474 }
2475 else
2476 {
2477 /*
2478 * Some 32-bit CPUs do not support CR8 load/store exiting as MOV CR8 is invalid on 32-bit Intel CPUs.
2479 * Set this control only for 64-bit guests.
2480 */
2481 if (pVM->hm.s.fAllow64BitGuests)
2482 {
2483 fVal |= VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT /* CR8 reads cause a VM-exit. */
2484 | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT; /* CR8 writes cause a VM-exit. */
2485 }
2486 }
2487
2488 /* Use MSR-bitmaps if supported by the CPU. */
2489 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
2490 {
2491 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS;
2492
2493 Assert(pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2494 Assert(!(pVCpu->hm.s.vmx.HCPhysMsrBitmap & 0xfff)); /* Bits 11:0 MBZ. */
2495 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_MSR_BITMAP_FULL, pVCpu->hm.s.vmx.HCPhysMsrBitmap);
2496 AssertRCReturn(rc, rc);
2497
2498 /*
2499 * The guest can access the following MSRs (read, write) without causing VM-exits; they are loaded/stored
2500 * automatically using dedicated fields in the VMCS.
2501 */
2502 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_CS, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2503 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_ESP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2504 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_SYSENTER_EIP, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2505 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2506 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_FS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2507#if HC_ARCH_BITS == 64
2508 /*
2509 * Set passthru permissions for the following MSRs (mandatory for VT-x) required for 64-bit guests.
2510 */
2511 if (pVM->hm.s.fAllow64BitGuests)
2512 {
2513 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_LSTAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2514 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_STAR, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2515 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_SF_MASK, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2516 hmR0VmxSetMsrPermission(pVCpu, MSR_K8_KERNEL_GS_BASE, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2517 }
2518#endif
2519 /*
2520 * The IA32_PRED_CMD MSR is write-only and has no state associated with it. We never need to intercept
2521 * access (writes need to be executed without exiting, reds will #GP-fault anyway).
2522 */
2523 if (pVM->cpum.ro.GuestFeatures.fIbpb)
2524 hmR0VmxSetMsrPermission(pVCpu, MSR_IA32_PRED_CMD, VMXMSREXIT_PASSTHRU_READ, VMXMSREXIT_PASSTHRU_WRITE);
2525
2526 /* Though MSR_IA32_PERF_GLOBAL_CTRL is saved/restored lazily, we want intercept reads/write to it for now. */
2527 }
2528
2529 /* Use the secondary processor-based VM-execution controls if supported by the CPU. */
2530 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2531 fVal |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
2532
2533 if ((fVal & fZap) != fVal)
2534 {
2535 LogRelFunc(("Invalid processor-based VM-execution controls combo! cpu=%#RX64 fVal=%#RX64 fZap=%#RX64\n",
2536 pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0, fVal, fZap));
2537 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_PROC_EXEC;
2538 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2539 }
2540
2541 /* Commit it to the VMCS and update our cache. */
2542 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, fVal);
2543 AssertRCReturn(rc, rc);
2544 pVCpu->hm.s.vmx.u32ProcCtls = fVal;
2545
2546 /* Set up secondary processor-based VM-execution controls if the CPU supports it. */
2547 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
2548 return hmR0VmxSetupProcCtls2(pVCpu);
2549
2550 /* Sanity check, should not really happen. */
2551 if (RT_UNLIKELY(pVM->hm.s.vmx.fUnrestrictedGuest))
2552 {
2553 LogRelFunc(("Unrestricted Guest enabled when secondary processor-based VM-execution controls not available\n"));
2554 pVCpu->hm.s.u32HMError = VMX_UFC_INVALID_UX_COMBO;
2555 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
2556 }
2557
2558 /* Old CPUs without secondary processor-based VM-execution controls would end up here. */
2559 return VINF_SUCCESS;
2560}
2561
2562
2563/**
2564 * Sets up miscellaneous (everything other than Pin & Processor-based
2565 * VM-execution) control fields in the VMCS.
2566 *
2567 * @returns VBox status code.
2568 * @param pVCpu The cross context virtual CPU structure.
2569 */
2570static int hmR0VmxSetupMiscCtls(PVMCPU pVCpu)
2571{
2572 AssertPtr(pVCpu);
2573
2574 int rc = VERR_GENERAL_FAILURE;
2575
2576 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2577#if 0
2578 /* All CR3 accesses cause VM-exits. Later we optimize CR3 accesses (see hmR0VmxExportGuestCR3AndCR4())*/
2579 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, 0);
2580 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, 0);
2581
2582 /*
2583 * Set MASK & MATCH to 0. VMX checks if GuestPFErrCode & MASK == MATCH. If equal (in our case it always is)
2584 * and if the X86_XCPT_PF bit in the exception bitmap is set it causes a VM-exit, if clear doesn't cause an exit.
2585 * We thus use the exception bitmap to control it rather than use both.
2586 */
2587 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, 0);
2588 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, 0);
2589
2590 /* All IO & IOIO instructions cause VM-exits. */
2591 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_A_FULL, 0);
2592 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_IO_BITMAP_B_FULL, 0);
2593
2594 /* Initialize the MSR-bitmap area. */
2595 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, 0);
2596 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, 0);
2597 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, 0);
2598 AssertRCReturn(rc, rc);
2599#endif
2600
2601 /* Setup MSR auto-load/store area. */
2602 Assert(pVCpu->hm.s.vmx.HCPhysGuestMsr);
2603 Assert(!(pVCpu->hm.s.vmx.HCPhysGuestMsr & 0xf)); /* Lower 4 bits MBZ. */
2604 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2605 rc |= VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL, pVCpu->hm.s.vmx.HCPhysGuestMsr);
2606 AssertRCReturn(rc, rc);
2607
2608 Assert(pVCpu->hm.s.vmx.HCPhysHostMsr);
2609 Assert(!(pVCpu->hm.s.vmx.HCPhysHostMsr & 0xf)); /* Lower 4 bits MBZ. */
2610 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL, pVCpu->hm.s.vmx.HCPhysHostMsr);
2611 AssertRCReturn(rc, rc);
2612
2613 /* Set VMCS link pointer. Reserved for future use, must be -1. Intel spec. 24.4 "Guest-State Area". */
2614 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, UINT64_C(0xffffffffffffffff));
2615 AssertRCReturn(rc, rc);
2616
2617 /* All fields are zero-initialized during allocation; but don't remove the commented block below. */
2618#if 0
2619 /* Setup debug controls */
2620 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, 0);
2621 rc |= VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0);
2622 AssertRCReturn(rc, rc);
2623#endif
2624
2625 return rc;
2626}
2627
2628
2629/**
2630 * Sets up the initial exception bitmap in the VMCS based on static conditions.
2631 *
2632 * We shall setup those exception intercepts that don't change during the
2633 * lifetime of the VM here. The rest are done dynamically while loading the
2634 * guest state.
2635 *
2636 * @returns VBox status code.
2637 * @param pVCpu The cross context virtual CPU structure.
2638 */
2639static int hmR0VmxInitXcptBitmap(PVMCPU pVCpu)
2640{
2641 AssertPtr(pVCpu);
2642
2643 uint32_t u32XcptBitmap;
2644
2645 /* Must always intercept #AC to prevent the guest from hanging the CPU. */
2646 u32XcptBitmap = RT_BIT_32(X86_XCPT_AC);
2647
2648 /* Because we need to maintain the DR6 state even when intercepting DRx reads
2649 and writes, and because recursive #DBs can cause the CPU hang, we must always
2650 intercept #DB. */
2651 u32XcptBitmap |= RT_BIT_32(X86_XCPT_DB);
2652
2653 /* Without Nested Paging, #PF must cause a VM-exit so we can sync our shadow page tables. */
2654 if (!pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
2655 u32XcptBitmap |= RT_BIT(X86_XCPT_PF);
2656
2657 /* Commit it to the VMCS. */
2658 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, u32XcptBitmap);
2659 AssertRCReturn(rc, rc);
2660
2661 /* Update our cache of the exception bitmap. */
2662 pVCpu->hm.s.vmx.u32XcptBitmap = u32XcptBitmap;
2663 return VINF_SUCCESS;
2664}
2665
2666
2667/**
2668 * Does per-VM VT-x initialization.
2669 *
2670 * @returns VBox status code.
2671 * @param pVM The cross context VM structure.
2672 */
2673VMMR0DECL(int) VMXR0InitVM(PVM pVM)
2674{
2675 LogFlowFunc(("pVM=%p\n", pVM));
2676
2677 int rc = hmR0VmxStructsAlloc(pVM);
2678 if (RT_FAILURE(rc))
2679 {
2680 LogRelFunc(("hmR0VmxStructsAlloc failed! rc=%Rrc\n", rc));
2681 return rc;
2682 }
2683
2684 return VINF_SUCCESS;
2685}
2686
2687
2688/**
2689 * Does per-VM VT-x termination.
2690 *
2691 * @returns VBox status code.
2692 * @param pVM The cross context VM structure.
2693 */
2694VMMR0DECL(int) VMXR0TermVM(PVM pVM)
2695{
2696 LogFlowFunc(("pVM=%p\n", pVM));
2697
2698#ifdef VBOX_WITH_CRASHDUMP_MAGIC
2699 if (pVM->hm.s.vmx.hMemObjScratch != NIL_RTR0MEMOBJ)
2700 ASMMemZero32(pVM->hm.s.vmx.pvScratch, PAGE_SIZE);
2701#endif
2702 hmR0VmxStructsFree(pVM);
2703 return VINF_SUCCESS;
2704}
2705
2706
2707/**
2708 * Sets up the VM for execution under VT-x.
2709 * This function is only called once per-VM during initialization.
2710 *
2711 * @returns VBox status code.
2712 * @param pVM The cross context VM structure.
2713 */
2714VMMR0DECL(int) VMXR0SetupVM(PVM pVM)
2715{
2716 AssertPtrReturn(pVM, VERR_INVALID_PARAMETER);
2717 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
2718
2719 LogFlowFunc(("pVM=%p\n", pVM));
2720
2721 /*
2722 * Without UnrestrictedGuest, pRealModeTSS and pNonPagingModeEPTPageTable *must* always be
2723 * allocated. We no longer support the highly unlikely case of UnrestrictedGuest without
2724 * pRealModeTSS, see hmR3InitFinalizeR0Intel().
2725 */
2726 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
2727 && ( !pVM->hm.s.vmx.pNonPagingModeEPTPageTable
2728 || !pVM->hm.s.vmx.pRealModeTSS))
2729 {
2730 LogRelFunc(("Invalid real-on-v86 state.\n"));
2731 return VERR_INTERNAL_ERROR;
2732 }
2733
2734 /* Initialize these always, see hmR3InitFinalizeR0().*/
2735 pVM->hm.s.vmx.enmFlushEpt = VMXFLUSHEPT_NONE;
2736 pVM->hm.s.vmx.enmFlushVpid = VMXFLUSHVPID_NONE;
2737
2738 /* Setup the tagged-TLB flush handlers. */
2739 int rc = hmR0VmxSetupTaggedTlb(pVM);
2740 if (RT_FAILURE(rc))
2741 {
2742 LogRelFunc(("hmR0VmxSetupTaggedTlb failed! rc=%Rrc\n", rc));
2743 return rc;
2744 }
2745
2746 /* Check if we can use the VMCS controls for swapping the EFER MSR. */
2747 Assert(!pVM->hm.s.vmx.fSupportsVmcsEfer);
2748#if HC_ARCH_BITS == 64
2749 if ( (pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1 & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
2750 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR)
2751 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR))
2752 {
2753 pVM->hm.s.vmx.fSupportsVmcsEfer = true;
2754 }
2755#endif
2756
2757 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
2758 RTCCUINTREG uHostCR4 = ASMGetCR4();
2759 if (RT_UNLIKELY(!(uHostCR4 & X86_CR4_VMXE)))
2760 return VERR_VMX_NOT_IN_VMX_ROOT_MODE;
2761
2762 for (VMCPUID i = 0; i < pVM->cCpus; i++)
2763 {
2764 PVMCPU pVCpu = &pVM->aCpus[i];
2765 AssertPtr(pVCpu);
2766 AssertPtr(pVCpu->hm.s.vmx.pvVmcs);
2767
2768 /* Log the VCPU pointers, useful for debugging SMP VMs. */
2769 Log4Func(("pVCpu=%p idCpu=%RU32\n", pVCpu, pVCpu->idCpu));
2770
2771 /* Set revision dword at the beginning of the VMCS structure. */
2772 *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs = MSR_IA32_VMX_BASIC_INFO_VMCS_ID(pVM->hm.s.vmx.Msrs.u64BasicInfo);
2773
2774 /* Initialize our VMCS region in memory, set the VMCS launch state to "clear". */
2775 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2776 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs failed! rc=%Rrc\n", rc),
2777 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2778
2779 /* Load this VMCS as the current VMCS. */
2780 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2781 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXActivateVmcs failed! rc=%Rrc\n", rc),
2782 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2783
2784 rc = hmR0VmxSetupPinCtls(pVCpu);
2785 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupPinCtls failed! rc=%Rrc\n", rc),
2786 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2787
2788 rc = hmR0VmxSetupProcCtls(pVCpu);
2789 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupProcCtls failed! rc=%Rrc\n", rc),
2790 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2791
2792 rc = hmR0VmxSetupMiscCtls(pVCpu);
2793 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxSetupMiscCtls failed! rc=%Rrc\n", rc),
2794 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2795
2796 rc = hmR0VmxInitXcptBitmap(pVCpu);
2797 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitXcptBitmap failed! rc=%Rrc\n", rc),
2798 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2799
2800#if HC_ARCH_BITS == 32
2801 rc = hmR0VmxInitVmcsReadCache(pVCpu);
2802 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: hmR0VmxInitVmcsReadCache failed! rc=%Rrc\n", rc),
2803 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2804#endif
2805
2806 /* Re-sync the CPU's internal data into our VMCS memory region & reset the launch state to "clear". */
2807 rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
2808 AssertLogRelMsgRCReturnStmt(rc, ("VMXR0SetupVM: VMXClearVmcs(2) failed! rc=%Rrc\n", rc),
2809 hmR0VmxUpdateErrorRecord(pVCpu, rc), rc);
2810
2811 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
2812
2813 hmR0VmxUpdateErrorRecord(pVCpu, rc);
2814 }
2815
2816 return VINF_SUCCESS;
2817}
2818
2819
2820/**
2821 * Saves the host control registers (CR0, CR3, CR4) into the host-state area in
2822 * the VMCS.
2823 *
2824 * @returns VBox status code.
2825 */
2826static int hmR0VmxExportHostControlRegs(void)
2827{
2828 RTCCUINTREG uReg = ASMGetCR0();
2829 int rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR0, uReg);
2830 AssertRCReturn(rc, rc);
2831
2832 uReg = ASMGetCR3();
2833 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR3, uReg);
2834 AssertRCReturn(rc, rc);
2835
2836 uReg = ASMGetCR4();
2837 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_CR4, uReg);
2838 AssertRCReturn(rc, rc);
2839 return rc;
2840}
2841
2842
2843/**
2844 * Saves the host segment registers and GDTR, IDTR, (TR, GS and FS bases) into
2845 * the host-state area in the VMCS.
2846 *
2847 * @returns VBox status code.
2848 * @param pVCpu The cross context virtual CPU structure.
2849 */
2850static int hmR0VmxExportHostSegmentRegs(PVMCPU pVCpu)
2851{
2852#if HC_ARCH_BITS == 64
2853/**
2854 * Macro for adjusting host segment selectors to satisfy VT-x's VM-entry
2855 * requirements. See hmR0VmxExportHostSegmentRegs().
2856 */
2857# define VMXLOCAL_ADJUST_HOST_SEG(seg, selValue) \
2858 if ((selValue) & (X86_SEL_RPL | X86_SEL_LDT)) \
2859 { \
2860 bool fValidSelector = true; \
2861 if ((selValue) & X86_SEL_LDT) \
2862 { \
2863 uint32_t uAttr = ASMGetSegAttr((selValue)); \
2864 fValidSelector = RT_BOOL(uAttr != UINT32_MAX && (uAttr & X86_DESC_P)); \
2865 } \
2866 if (fValidSelector) \
2867 { \
2868 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_##seg; \
2869 pVCpu->hm.s.vmx.RestoreHost.uHostSel##seg = (selValue); \
2870 } \
2871 (selValue) = 0; \
2872 }
2873
2874 /*
2875 * If we've executed guest code using VT-x, the host-state bits will be messed up. We
2876 * should -not- save the messed up state without restoring the original host-state,
2877 * see @bugref{7240}.
2878 *
2879 * This apparently can happen (most likely the FPU changes), deal with it rather than
2880 * asserting. Was observed booting Solaris 10u10 32-bit guest.
2881 */
2882 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
2883 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
2884 {
2885 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags,
2886 pVCpu->idCpu));
2887 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
2888 }
2889 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
2890#else
2891 RT_NOREF(pVCpu);
2892#endif
2893
2894 /*
2895 * Host DS, ES, FS and GS segment registers.
2896 */
2897#if HC_ARCH_BITS == 64
2898 RTSEL uSelDS = ASMGetDS();
2899 RTSEL uSelES = ASMGetES();
2900 RTSEL uSelFS = ASMGetFS();
2901 RTSEL uSelGS = ASMGetGS();
2902#else
2903 RTSEL uSelDS = 0;
2904 RTSEL uSelES = 0;
2905 RTSEL uSelFS = 0;
2906 RTSEL uSelGS = 0;
2907#endif
2908
2909 /*
2910 * Host CS and SS segment registers.
2911 */
2912 RTSEL uSelCS = ASMGetCS();
2913 RTSEL uSelSS = ASMGetSS();
2914
2915 /*
2916 * Host TR segment register.
2917 */
2918 RTSEL uSelTR = ASMGetTR();
2919
2920#if HC_ARCH_BITS == 64
2921 /*
2922 * Determine if the host segment registers are suitable for VT-x. Otherwise use zero to
2923 * gain VM-entry and restore them before we get preempted.
2924 *
2925 * See Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers".
2926 */
2927 VMXLOCAL_ADJUST_HOST_SEG(DS, uSelDS);
2928 VMXLOCAL_ADJUST_HOST_SEG(ES, uSelES);
2929 VMXLOCAL_ADJUST_HOST_SEG(FS, uSelFS);
2930 VMXLOCAL_ADJUST_HOST_SEG(GS, uSelGS);
2931# undef VMXLOCAL_ADJUST_HOST_SEG
2932#endif
2933
2934 /* Verification based on Intel spec. 26.2.3 "Checks on Host Segment and Descriptor-Table Registers" */
2935 Assert(!(uSelCS & X86_SEL_RPL)); Assert(!(uSelCS & X86_SEL_LDT));
2936 Assert(!(uSelSS & X86_SEL_RPL)); Assert(!(uSelSS & X86_SEL_LDT));
2937 Assert(!(uSelDS & X86_SEL_RPL)); Assert(!(uSelDS & X86_SEL_LDT));
2938 Assert(!(uSelES & X86_SEL_RPL)); Assert(!(uSelES & X86_SEL_LDT));
2939 Assert(!(uSelFS & X86_SEL_RPL)); Assert(!(uSelFS & X86_SEL_LDT));
2940 Assert(!(uSelGS & X86_SEL_RPL)); Assert(!(uSelGS & X86_SEL_LDT));
2941 Assert(!(uSelTR & X86_SEL_RPL)); Assert(!(uSelTR & X86_SEL_LDT));
2942 Assert(uSelCS);
2943 Assert(uSelTR);
2944
2945 /* Assertion is right but we would not have updated u32ExitCtls yet. */
2946#if 0
2947 if (!(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE))
2948 Assert(uSelSS != 0);
2949#endif
2950
2951 /* Write these host selector fields into the host-state area in the VMCS. */
2952 int rc = VMXWriteVmcs32(VMX_VMCS16_HOST_CS_SEL, uSelCS);
2953 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_SS_SEL, uSelSS);
2954#if HC_ARCH_BITS == 64
2955 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_DS_SEL, uSelDS);
2956 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_ES_SEL, uSelES);
2957 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_FS_SEL, uSelFS);
2958 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_GS_SEL, uSelGS);
2959#else
2960 NOREF(uSelDS);
2961 NOREF(uSelES);
2962 NOREF(uSelFS);
2963 NOREF(uSelGS);
2964#endif
2965 rc |= VMXWriteVmcs32(VMX_VMCS16_HOST_TR_SEL, uSelTR);
2966 AssertRCReturn(rc, rc);
2967
2968 /*
2969 * Host GDTR and IDTR.
2970 */
2971 RTGDTR Gdtr;
2972 RTIDTR Idtr;
2973 RT_ZERO(Gdtr);
2974 RT_ZERO(Idtr);
2975 ASMGetGDTR(&Gdtr);
2976 ASMGetIDTR(&Idtr);
2977 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, Gdtr.pGdt);
2978 rc |= VMXWriteVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, Idtr.pIdt);
2979 AssertRCReturn(rc, rc);
2980
2981#if HC_ARCH_BITS == 64
2982 /*
2983 * Determine if we need to manually need to restore the GDTR and IDTR limits as VT-x zaps
2984 * them to the maximum limit (0xffff) on every VM-exit.
2985 */
2986 if (Gdtr.cbGdt != 0xffff)
2987 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDTR;
2988
2989 /*
2990 * IDT limit is effectively capped at 0xfff. (See Intel spec. 6.14.1 "64-Bit Mode IDT" and
2991 * Intel spec. 6.2 "Exception and Interrupt Vectors".) Therefore if the host has the limit
2992 * as 0xfff, VT-x bloating the limit to 0xffff shouldn't cause any different CPU behavior.
2993 * However, several hosts either insists on 0xfff being the limit (Windows Patch Guard) or
2994 * uses the limit for other purposes (darwin puts the CPU ID in there but botches sidt
2995 * alignment in at least one consumer). So, we're only allowing the IDTR.LIMIT to be left
2996 * at 0xffff on hosts where we are sure it won't cause trouble.
2997 */
2998# if defined(RT_OS_LINUX) || defined(RT_OS_SOLARIS)
2999 if (Idtr.cbIdt < 0x0fff)
3000# else
3001 if (Idtr.cbIdt != 0xffff)
3002# endif
3003 {
3004 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_IDTR;
3005 AssertCompile(sizeof(Idtr) == sizeof(X86XDTR64));
3006 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostIdtr, &Idtr, sizeof(X86XDTR64));
3007 }
3008#endif
3009
3010 /*
3011 * Host TR base. Verify that TR selector doesn't point past the GDT. Masking off the TI
3012 * and RPL bits is effectively what the CPU does for "scaling by 8". TI is always 0 and
3013 * RPL should be too in most cases.
3014 */
3015 AssertMsgReturn((uSelTR | X86_SEL_RPL_LDT) <= Gdtr.cbGdt,
3016 ("TR selector exceeds limit. TR=%RTsel cbGdt=%#x\n", uSelTR, Gdtr.cbGdt), VERR_VMX_INVALID_HOST_STATE);
3017
3018 PCX86DESCHC pDesc = (PCX86DESCHC)(Gdtr.pGdt + (uSelTR & X86_SEL_MASK));
3019#if HC_ARCH_BITS == 64
3020 uintptr_t uTRBase = X86DESC64_BASE(pDesc);
3021
3022 /*
3023 * VT-x unconditionally restores the TR limit to 0x67 and type to 11 (32-bit busy TSS) on
3024 * all VM-exits. The type is the same for 64-bit busy TSS[1]. The limit needs manual
3025 * restoration if the host has something else. Task switching is not supported in 64-bit
3026 * mode[2], but the limit still matters as IOPM is supported in 64-bit mode. Restoring the
3027 * limit lazily while returning to ring-3 is safe because IOPM is not applicable in ring-0.
3028 *
3029 * [1] See Intel spec. 3.5 "System Descriptor Types".
3030 * [2] See Intel spec. 7.2.3 "TSS Descriptor in 64-bit mode".
3031 */
3032 PVM pVM = pVCpu->CTX_SUFF(pVM);
3033 Assert(pDesc->System.u4Type == 11);
3034 if ( pDesc->System.u16LimitLow != 0x67
3035 || pDesc->System.u4LimitHigh)
3036 {
3037 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_SEL_TR;
3038 /* If the host has made GDT read-only, we would need to temporarily toggle CR0.WP before writing the GDT. */
3039 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_READ_ONLY)
3040 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_READ_ONLY;
3041 pVCpu->hm.s.vmx.RestoreHost.uHostSelTR = uSelTR;
3042 }
3043
3044 /*
3045 * Store the GDTR as we need it when restoring the GDT and while restoring the TR.
3046 */
3047 if (pVCpu->hm.s.vmx.fRestoreHostFlags & (VMX_RESTORE_HOST_GDTR | VMX_RESTORE_HOST_SEL_TR))
3048 {
3049 AssertCompile(sizeof(Gdtr) == sizeof(X86XDTR64));
3050 memcpy(&pVCpu->hm.s.vmx.RestoreHost.HostGdtr, &Gdtr, sizeof(X86XDTR64));
3051 if (pVM->hm.s.fHostKernelFeatures & SUPKERNELFEATURES_GDT_NEED_WRITABLE)
3052 {
3053 /* The GDT is read-only but the writable GDT is available. */
3054 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_GDT_NEED_WRITABLE;
3055 pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.cb = Gdtr.cbGdt;
3056 rc = SUPR0GetCurrentGdtRw(&pVCpu->hm.s.vmx.RestoreHost.HostGdtrRw.uAddr);
3057 AssertRCReturn(rc, rc);
3058 }
3059 }
3060#else
3061 uintptr_t uTRBase = X86DESC_BASE(pDesc);
3062#endif
3063 rc = VMXWriteVmcsHstN(VMX_VMCS_HOST_TR_BASE, uTRBase);
3064 AssertRCReturn(rc, rc);
3065
3066 /*
3067 * Host FS base and GS base.
3068 */
3069#if HC_ARCH_BITS == 64
3070 uint64_t u64FSBase = ASMRdMsr(MSR_K8_FS_BASE);
3071 uint64_t u64GSBase = ASMRdMsr(MSR_K8_GS_BASE);
3072 rc = VMXWriteVmcs64(VMX_VMCS_HOST_FS_BASE, u64FSBase);
3073 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_GS_BASE, u64GSBase);
3074 AssertRCReturn(rc, rc);
3075
3076 /* Store the base if we have to restore FS or GS manually as we need to restore the base as well. */
3077 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_FS)
3078 pVCpu->hm.s.vmx.RestoreHost.uHostFSBase = u64FSBase;
3079 if (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_SEL_GS)
3080 pVCpu->hm.s.vmx.RestoreHost.uHostGSBase = u64GSBase;
3081#endif
3082 return VINF_SUCCESS;
3083}
3084
3085
3086/**
3087 * Exports certain host MSRs in the VM-exit MSR-load area and some in the
3088 * host-state area of the VMCS.
3089 *
3090 * Theses MSRs will be automatically restored on the host after every successful
3091 * VM-exit.
3092 *
3093 * @returns VBox status code.
3094 * @param pVCpu The cross context virtual CPU structure.
3095 *
3096 * @remarks No-long-jump zone!!!
3097 */
3098static int hmR0VmxExportHostMsrs(PVMCPU pVCpu)
3099{
3100 AssertPtr(pVCpu);
3101 AssertPtr(pVCpu->hm.s.vmx.pvHostMsr);
3102
3103 /*
3104 * Save MSRs that we restore lazily (due to preemption or transition to ring-3)
3105 * rather than swapping them on every VM-entry.
3106 */
3107 hmR0VmxLazySaveHostMsrs(pVCpu);
3108
3109 /*
3110 * Host Sysenter MSRs.
3111 */
3112 int rc = VMXWriteVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, ASMRdMsr_Low(MSR_IA32_SYSENTER_CS));
3113#if HC_ARCH_BITS == 32
3114 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr_Low(MSR_IA32_SYSENTER_ESP));
3115 rc |= VMXWriteVmcs32(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr_Low(MSR_IA32_SYSENTER_EIP));
3116#else
3117 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, ASMRdMsr(MSR_IA32_SYSENTER_ESP));
3118 rc |= VMXWriteVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, ASMRdMsr(MSR_IA32_SYSENTER_EIP));
3119#endif
3120 AssertRCReturn(rc, rc);
3121
3122 /*
3123 * Host EFER MSR.
3124 *
3125 * If the CPU supports the newer VMCS controls for managing EFER, use it. Otherwise it's
3126 * done as part of auto-load/store MSR area in the VMCS, see hmR0VmxExportGuestMsrs().
3127 */
3128 PVM pVM = pVCpu->CTX_SUFF(pVM);
3129 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
3130 {
3131 rc = VMXWriteVmcs64(VMX_VMCS64_HOST_EFER_FULL, pVM->hm.s.vmx.u64HostEfer);
3132 AssertRCReturn(rc, rc);
3133 }
3134
3135 /** @todo IA32_PERF_GLOBALCTRL, IA32_PAT also see hmR0VmxExportGuestExitCtls(). */
3136
3137 return VINF_SUCCESS;
3138}
3139
3140
3141/**
3142 * Figures out if we need to swap the EFER MSR which is particularly expensive.
3143 *
3144 * We check all relevant bits. For now, that's everything besides LMA/LME, as
3145 * these two bits are handled by VM-entry, see hmR0VmxExportGuestExitCtls() and
3146 * hmR0VMxExportGuestEntryCtls().
3147 *
3148 * @returns true if we need to load guest EFER, false otherwise.
3149 * @param pVCpu The cross context virtual CPU structure.
3150 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3151 * out-of-sync. Make sure to update the required fields
3152 * before using them.
3153 *
3154 * @remarks Requires EFER, CR4.
3155 * @remarks No-long-jump zone!!!
3156 */
3157static bool hmR0VmxShouldSwapEferMsr(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3158{
3159#ifdef HMVMX_ALWAYS_SWAP_EFER
3160 return true;
3161#endif
3162
3163#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
3164 /* For 32-bit hosts running 64-bit guests, we always swap EFER in the world-switcher. Nothing to do here. */
3165 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3166 return false;
3167#endif
3168
3169 PVM pVM = pVCpu->CTX_SUFF(pVM);
3170 uint64_t u64HostEfer = pVM->hm.s.vmx.u64HostEfer;
3171 uint64_t u64GuestEfer = pMixedCtx->msrEFER;
3172
3173 /*
3174 * For 64-bit guests, if EFER.SCE bit differs, we need to swap EFER to ensure that the
3175 * guest's SYSCALL behaviour isn't broken, see @bugref{7386}.
3176 */
3177 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
3178 && (u64GuestEfer & MSR_K6_EFER_SCE) != (u64HostEfer & MSR_K6_EFER_SCE))
3179 {
3180 return true;
3181 }
3182
3183 /*
3184 * If the guest uses PAE and EFER.NXE bit differs, we need to swap EFER as it
3185 * affects guest paging. 64-bit paging implies CR4.PAE as well.
3186 * See Intel spec. 4.5 "IA-32e Paging" and Intel spec. 4.1.1 "Three Paging Modes".
3187 */
3188 if ( (pMixedCtx->cr4 & X86_CR4_PAE)
3189 && (pMixedCtx->cr0 & X86_CR0_PG)
3190 && (u64GuestEfer & MSR_K6_EFER_NXE) != (u64HostEfer & MSR_K6_EFER_NXE))
3191 {
3192 /* Assert that host is PAE capable. */
3193 Assert(pVM->hm.s.cpuid.u32AMDFeatureEDX & X86_CPUID_EXT_FEATURE_EDX_NX);
3194 return true;
3195 }
3196
3197 return false;
3198}
3199
3200
3201/**
3202 * Exports the guest state with appropriate VM-entry controls in the VMCS.
3203 *
3204 * These controls can affect things done on VM-exit; e.g. "load debug controls",
3205 * see Intel spec. 24.8.1 "VM-entry controls".
3206 *
3207 * @returns VBox status code.
3208 * @param pVCpu The cross context virtual CPU structure.
3209 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3210 * out-of-sync. Make sure to update the required fields
3211 * before using them.
3212 *
3213 * @remarks Requires EFER.
3214 * @remarks No-long-jump zone!!!
3215 */
3216static int hmR0VmxExportGuestEntryCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3217{
3218 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_ENTRY_CTLS)
3219 {
3220 PVM pVM = pVCpu->CTX_SUFF(pVM);
3221 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0; /* Bits set here must be set in the VMCS. */
3222 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxEntry.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3223
3224 /* Load debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x capable CPUs only supports the 1-setting of this bit. */
3225 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG;
3226
3227 /* Set if the guest is in long mode. This will set/clear the EFER.LMA bit on VM-entry. */
3228 if (CPUMIsGuestInLongModeEx(pMixedCtx))
3229 {
3230 fVal |= VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST;
3231 Log4Func(("VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST\n"));
3232 }
3233 else
3234 Assert(!(fVal & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST));
3235
3236 /* If the CPU supports the newer VMCS controls for managing guest/host EFER, use it. */
3237 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3238 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3239 {
3240 fVal |= VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR;
3241 Log4Func(("VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR\n"));
3242 }
3243
3244 /*
3245 * The following should -not- be set (since we're not in SMM mode):
3246 * - VMX_VMCS_CTRL_ENTRY_ENTRY_SMM
3247 * - VMX_VMCS_CTRL_ENTRY_DEACTIVATE_DUALMON
3248 */
3249
3250 /** @todo VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR,
3251 * VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR. */
3252
3253 if ((fVal & fZap) != fVal)
3254 {
3255 Log4Func(("Invalid VM-entry controls combo! Cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3256 pVM->hm.s.vmx.Msrs.VmxEntry.n.disallowed0, fVal, fZap));
3257 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_ENTRY;
3258 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3259 }
3260
3261 /* Commit it to the VMCS and update our cache. */
3262 if (pVCpu->hm.s.vmx.u32EntryCtls != fVal)
3263 {
3264 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY, fVal);
3265 AssertRCReturn(rc, rc);
3266 pVCpu->hm.s.vmx.u32EntryCtls = fVal;
3267 }
3268
3269 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_ENTRY_CTLS);
3270 }
3271 return VINF_SUCCESS;
3272}
3273
3274
3275/**
3276 * Exports the guest state with appropriate VM-exit controls in the VMCS.
3277 *
3278 * @returns VBox status code.
3279 * @param pVCpu The cross context virtual CPU structure.
3280 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3281 * out-of-sync. Make sure to update the required fields
3282 * before using them.
3283 *
3284 * @remarks Requires EFER.
3285 */
3286static int hmR0VmxExportGuestExitCtls(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3287{
3288 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_EXIT_CTLS)
3289 {
3290 PVM pVM = pVCpu->CTX_SUFF(pVM);
3291 uint32_t fVal = pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0; /* Bits set here must be set in the VMCS. */
3292 uint32_t const fZap = pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1; /* Bits cleared here must be cleared in the VMCS. */
3293
3294 /* Save debug controls (DR7 & IA32_DEBUGCTL_MSR). The first VT-x CPUs only supported the 1-setting of this bit. */
3295 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_DEBUG;
3296
3297 /*
3298 * Set the host long mode active (EFER.LMA) bit (which Intel calls "Host address-space size") if necessary.
3299 * On VM-exit, VT-x sets both the host EFER.LMA and EFER.LME bit to this value. See assertion in
3300 * hmR0VmxExportHostMsrs().
3301 */
3302#if HC_ARCH_BITS == 64
3303 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3304 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3305#else
3306 Assert( pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64
3307 || pVCpu->hm.s.vmx.pfnStartVM == VMXR0StartVM32);
3308 /* Set the host address-space size based on the switcher, not guest state. See @bugref{8432}. */
3309 if (pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64)
3310 {
3311 /* The switcher returns to long mode, EFER is managed by the switcher. */
3312 fVal |= VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE;
3313 Log4Func(("VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE\n"));
3314 }
3315 else
3316 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_HOST_ADDR_SPACE_SIZE));
3317#endif
3318
3319 /* If the newer VMCS fields for managing EFER exists, use it. */
3320 if ( pVM->hm.s.vmx.fSupportsVmcsEfer
3321 && hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
3322 {
3323 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR
3324 | VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR;
3325 Log4Func(("VMX_VMCS_CTRL_EXIT_SAVE_GUEST_EFER_MSR and VMX_VMCS_CTRL_EXIT_LOAD_HOST_EFER_MSR\n"));
3326 }
3327
3328 /* Don't acknowledge external interrupts on VM-exit. We want to let the host do that. */
3329 Assert(!(fVal & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT));
3330
3331 /** @todo VMX_VMCS_CTRL_EXIT_LOAD_PERF_MSR,
3332 * VMX_VMCS_CTRL_EXIT_SAVE_GUEST_PAT_MSR,
3333 * VMX_VMCS_CTRL_EXIT_LOAD_HOST_PAT_MSR. */
3334
3335 /* Enable saving of the VMX preemption timer value on VM-exit. */
3336 if ( pVM->hm.s.vmx.fUsePreemptTimer
3337 && (pVM->hm.s.vmx.Msrs.VmxExit.n.allowed1 & VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER))
3338 fVal |= VMX_VMCS_CTRL_EXIT_SAVE_VMX_PREEMPT_TIMER;
3339
3340 if ((fVal & fZap) != fVal)
3341 {
3342 LogRelFunc(("Invalid VM-exit controls combo! cpu=%RX64 fVal=%RX64 fZap=%RX64\n",
3343 pVM->hm.s.vmx.Msrs.VmxExit.n.disallowed0, fVal, fZap));
3344 pVCpu->hm.s.u32HMError = VMX_UFC_CTRL_EXIT;
3345 return VERR_HM_UNSUPPORTED_CPU_FEATURE_COMBO;
3346 }
3347
3348 /* Commit it to the VMCS and update our cache. */
3349 if (pVCpu->hm.s.vmx.u32ExitCtls != fVal)
3350 {
3351 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXIT, fVal);
3352 AssertRCReturn(rc, rc);
3353 pVCpu->hm.s.vmx.u32ExitCtls = fVal;
3354 }
3355
3356 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_EXIT_CTLS);
3357 }
3358 return VINF_SUCCESS;
3359}
3360
3361
3362/**
3363 * Sets the TPR threshold in the VMCS.
3364 *
3365 * @returns VBox status code.
3366 * @param pVCpu The cross context virtual CPU structure.
3367 * @param u32TprThreshold The TPR threshold (task-priority class only).
3368 */
3369DECLINLINE(int) hmR0VmxApicSetTprThreshold(PVMCPU pVCpu, uint32_t u32TprThreshold)
3370{
3371 Assert(!(u32TprThreshold & 0xfffffff0)); /* Bits 31:4 MBZ. */
3372 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW); RT_NOREF_PV(pVCpu);
3373 return VMXWriteVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, u32TprThreshold);
3374}
3375
3376
3377/**
3378 * Exports the guest APIC TPR state into the VMCS.
3379 *
3380 * @returns VBox status code.
3381 * @param pVCpu The cross context virtual CPU structure.
3382 *
3383 * @remarks No-long-jump zone!!!
3384 */
3385static int hmR0VmxExportGuestApicTpr(PVMCPU pVCpu)
3386{
3387 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_APIC_TPR)
3388 {
3389 if ( PDMHasApic(pVCpu->CTX_SUFF(pVM))
3390 && APICIsEnabled(pVCpu))
3391 {
3392 /*
3393 * Setup TPR shadowing.
3394 */
3395 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
3396 {
3397 Assert(pVCpu->hm.s.vmx.HCPhysVirtApic);
3398
3399 bool fPendingIntr = false;
3400 uint8_t u8Tpr = 0;
3401 uint8_t u8PendingIntr = 0;
3402 int rc = APICGetTpr(pVCpu, &u8Tpr, &fPendingIntr, &u8PendingIntr);
3403 AssertRCReturn(rc, rc);
3404
3405 /*
3406 * If there are interrupts pending but masked by the TPR, instruct VT-x to
3407 * cause a TPR-below-threshold VM-exit when the guest lowers its TPR below the
3408 * priority of the pending interrupt so we can deliver the interrupt. If there
3409 * are no interrupts pending, set threshold to 0 to not cause any
3410 * TPR-below-threshold VM-exits.
3411 */
3412 pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR] = u8Tpr;
3413 uint32_t u32TprThreshold = 0;
3414 if (fPendingIntr)
3415 {
3416 /* Bits 3:0 of the TPR threshold field correspond to bits 7:4 of the TPR (which is the Task-Priority Class). */
3417 const uint8_t u8PendingPriority = u8PendingIntr >> 4;
3418 const uint8_t u8TprPriority = u8Tpr >> 4;
3419 if (u8PendingPriority <= u8TprPriority)
3420 u32TprThreshold = u8PendingPriority;
3421 }
3422
3423 rc = hmR0VmxApicSetTprThreshold(pVCpu, u32TprThreshold);
3424 AssertRCReturn(rc, rc);
3425 }
3426 }
3427 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_APIC_TPR);
3428 }
3429 return VINF_SUCCESS;
3430}
3431
3432
3433/**
3434 * Gets the guest's interruptibility-state ("interrupt shadow" as AMD calls it).
3435 *
3436 * @returns Guest's interruptibility-state.
3437 * @param pVCpu The cross context virtual CPU structure.
3438 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3439 * out-of-sync. Make sure to update the required fields
3440 * before using them.
3441 *
3442 * @remarks No-long-jump zone!!!
3443 */
3444static uint32_t hmR0VmxGetGuestIntrState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
3445{
3446 /*
3447 * Check if we should inhibit interrupt delivery due to instructions like STI and MOV SS.
3448 */
3449 uint32_t fIntrState = 0;
3450 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3451 {
3452 /* If inhibition is active, RIP & RFLAGS should've been accessed
3453 (i.e. read previously from the VMCS or from ring-3). */
3454#ifdef VBOX_STRICT
3455 uint64_t const fExtrn = ASMAtomicUoReadU64(&pMixedCtx->fExtrn);
3456 AssertMsg(!(fExtrn & (CPUMCTX_EXTRN_RIP | CPUMCTX_EXTRN_RFLAGS)), ("%#x\n", fExtrn));
3457#endif
3458 if (pMixedCtx->rip == EMGetInhibitInterruptsPC(pVCpu))
3459 {
3460 if (pMixedCtx->eflags.Bits.u1IF)
3461 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
3462 else
3463 fIntrState = VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS;
3464 }
3465 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
3466 {
3467 /*
3468 * We can clear the inhibit force flag as even if we go back to the recompiler
3469 * without executing guest code in VT-x, the flag's condition to be cleared is
3470 * met and thus the cleared state is correct.
3471 */
3472 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
3473 }
3474 }
3475
3476 /*
3477 * NMIs to the guest are blocked after an NMI is injected until the guest executes an IRET. We only
3478 * bother with virtual-NMI blocking when we have support for virtual NMIs in the CPU, otherwise
3479 * setting this would block host-NMIs and IRET will not clear the blocking.
3480 *
3481 * See Intel spec. 26.6.1 "Interruptibility state". See @bugref{7445}.
3482 */
3483 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
3484 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
3485 {
3486 fIntrState |= VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI;
3487 }
3488
3489 return fIntrState;
3490}
3491
3492
3493/**
3494 * Exports the guest's interruptibility-state into the guest-state area in the
3495 * VMCS.
3496 *
3497 * @returns VBox status code.
3498 * @param pVCpu The cross context virtual CPU structure.
3499 * @param fIntrState The interruptibility-state to set.
3500 */
3501static int hmR0VmxExportGuestIntrState(PVMCPU pVCpu, uint32_t fIntrState)
3502{
3503 NOREF(pVCpu);
3504 AssertMsg(!(fIntrState & 0xfffffff0), ("%#x\n", fIntrState)); /* Bits 31:4 MBZ. */
3505 Assert((fIntrState & 0x3) != 0x3); /* Block-by-STI and MOV SS cannot be simultaneously set. */
3506 return VMXWriteVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, fIntrState);
3507}
3508
3509
3510/**
3511 * Exports the exception intercepts required for guest execution in the VMCS.
3512 *
3513 * @returns VBox status code.
3514 * @param pVCpu The cross context virtual CPU structure.
3515 *
3516 * @remarks No-long-jump zone!!!
3517 */
3518static int hmR0VmxExportGuestXcptIntercepts(PVMCPU pVCpu)
3519{
3520 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS)
3521 {
3522 /* The remaining exception intercepts are handled elsewhere, e.g. in hmR0VmxExportSharedCR0(). */
3523 if (pVCpu->hm.s.fGIMTrapXcptUD)
3524 pVCpu->hm.s.vmx.u32XcptBitmap |= RT_BIT(X86_XCPT_UD);
3525#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3526 else
3527 pVCpu->hm.s.vmx.u32XcptBitmap &= ~RT_BIT(X86_XCPT_UD);
3528#endif
3529
3530 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_AC));
3531 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
3532
3533 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
3534 AssertRCReturn(rc, rc);
3535
3536 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3537 Log4Func(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP=%#RX64\n", pVCpu->hm.s.vmx.u32XcptBitmap));
3538 }
3539 return VINF_SUCCESS;
3540}
3541
3542
3543/**
3544 * Exports the guest's RIP into the guest-state area in the VMCS.
3545 *
3546 * @returns VBox status code.
3547 * @param pVCpu The cross context virtual CPU structure.
3548 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3549 * out-of-sync. Make sure to update the required fields
3550 * before using them.
3551 *
3552 * @remarks No-long-jump zone!!!
3553 */
3554static int hmR0VmxExportGuestRip(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3555{
3556 int rc = VINF_SUCCESS;
3557 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
3558 {
3559 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RIP, pMixedCtx->rip);
3560 AssertRCReturn(rc, rc);
3561
3562 /* Update the exit history entry with the correct CS.BASE + RIP or just RIP. */
3563 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CS)
3564 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
3565 else
3566 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->rip, false);
3567
3568 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RIP);
3569 Log4Func(("RIP=%#RX64\n", pMixedCtx->rip));
3570 }
3571 return rc;
3572}
3573
3574
3575/**
3576 * Exports the guest's RSP into the guest-state area in the VMCS.
3577 *
3578 * @returns VBox status code.
3579 * @param pVCpu The cross context virtual CPU structure.
3580 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3581 * out-of-sync. Make sure to update the required fields
3582 * before using them.
3583 *
3584 * @remarks No-long-jump zone!!!
3585 */
3586static int hmR0VmxExportGuestRsp(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3587{
3588 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RSP)
3589 {
3590 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_RSP, pMixedCtx->rsp);
3591 AssertRCReturn(rc, rc);
3592
3593 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RSP);
3594 }
3595 return VINF_SUCCESS;
3596}
3597
3598
3599/**
3600 * Exports the guest's RFLAGS into the guest-state area in the VMCS.
3601 *
3602 * @returns VBox status code.
3603 * @param pVCpu The cross context virtual CPU structure.
3604 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3605 * out-of-sync. Make sure to update the required fields
3606 * before using them.
3607 *
3608 * @remarks No-long-jump zone!!!
3609 */
3610static int hmR0VmxExportGuestRflags(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3611{
3612 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RFLAGS)
3613 {
3614 /* Intel spec. 2.3.1 "System Flags and Fields in IA-32e Mode" claims the upper 32-bits of RFLAGS are reserved (MBZ).
3615 Let us assert it as such and use 32-bit VMWRITE. */
3616 Assert(!RT_HI_U32(pMixedCtx->rflags.u64));
3617 X86EFLAGS fEFlags = pMixedCtx->eflags;
3618 Assert(fEFlags.u32 & X86_EFL_RA1_MASK);
3619 Assert(!(fEFlags.u32 & ~(X86_EFL_1 | X86_EFL_LIVE_MASK)));
3620
3621 /*
3622 * If we're emulating real-mode using Virtual 8086 mode, save the real-mode eflags so
3623 * we can restore them on VM-exit. Modify the real-mode guest's eflags so that VT-x
3624 * can run the real-mode guest code under Virtual 8086 mode.
3625 */
3626 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3627 {
3628 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
3629 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
3630 pVCpu->hm.s.vmx.RealMode.Eflags.u32 = fEFlags.u32; /* Save the original eflags of the real-mode guest. */
3631 fEFlags.Bits.u1VM = 1; /* Set the Virtual 8086 mode bit. */
3632 fEFlags.Bits.u2IOPL = 0; /* Change IOPL to 0, otherwise certain instructions won't fault. */
3633 }
3634
3635 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_RFLAGS, fEFlags.u32);
3636 AssertRCReturn(rc, rc);
3637
3638 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_RFLAGS);
3639 Log4Func(("EFlags=%#RX32\n", fEFlags.u32));
3640 }
3641 return VINF_SUCCESS;
3642}
3643
3644
3645/**
3646 * Exports the guest CR0 control register into the guest-state area in the VMCS.
3647 *
3648 * The guest FPU state is always pre-loaded hence we don't need to bother about
3649 * sharing FPU related CR0 bits between the guest and host.
3650 *
3651 * @returns VBox status code.
3652 * @param pVCpu The cross context virtual CPU structure.
3653 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3654 * out-of-sync. Make sure to update the required fields
3655 * before using them.
3656 *
3657 * @remarks No-long-jump zone!!!
3658 */
3659static int hmR0VmxExportGuestCR0(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3660{
3661 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR0)
3662 {
3663 PVM pVM = pVCpu->CTX_SUFF(pVM);
3664 Assert(!RT_HI_U32(pMixedCtx->cr0));
3665 uint32_t const uShadowCR0 = pMixedCtx->cr0;
3666 uint32_t uGuestCR0 = pMixedCtx->cr0;
3667
3668 /*
3669 * Setup VT-x's view of the guest CR0.
3670 * Minimize VM-exits due to CR3 changes when we have NestedPaging.
3671 */
3672 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
3673 if (pVM->hm.s.fNestedPaging)
3674 {
3675 if (CPUMIsGuestPagingEnabled(pVCpu))
3676 {
3677 /* The guest has paging enabled, let it access CR3 without causing a VM-exit if supported. */
3678 uProcCtls &= ~( VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3679 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT);
3680 }
3681 else
3682 {
3683 /* The guest doesn't have paging enabled, make CR3 access cause a VM-exit to update our shadow. */
3684 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT
3685 | VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3686 }
3687
3688 /* If we have unrestricted guest execution, we never have to intercept CR3 reads. */
3689 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3690 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT;
3691 }
3692 else
3693 {
3694 /* Guest CPL 0 writes to its read-only pages should cause a #PF VM-exit. */
3695 uGuestCR0 |= X86_CR0_WP;
3696 }
3697
3698 /*
3699 * Guest FPU bits.
3700 *
3701 * Since we pre-load the guest FPU always before VM-entry there is no need to track lazy state
3702 * using CR0.TS.
3703 *
3704 * Intel spec. 23.8 "Restrictions on VMX operation" mentions that CR0.NE bit must always be
3705 * set on the first CPUs to support VT-x and no mention of with regards to UX in VM-entry checks.
3706 */
3707 uGuestCR0 |= X86_CR0_NE;
3708
3709 /* If CR0.NE isn't set, we need to intercept #MF exceptions and report them to the guest differently. */
3710 bool const fInterceptMF = !(uShadowCR0 & X86_CR0_NE);
3711
3712 /*
3713 * Update exception intercepts.
3714 */
3715 uint32_t uXcptBitmap = pVCpu->hm.s.vmx.u32XcptBitmap;
3716 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3717 {
3718 Assert(PDMVmmDevHeapIsEnabled(pVM));
3719 Assert(pVM->hm.s.vmx.pRealModeTSS);
3720 uXcptBitmap |= HMVMX_REAL_MODE_XCPT_MASK;
3721 }
3722 else
3723 {
3724 /* For now, cleared here as mode-switches can happen outside HM/VT-x. See @bugref{7626#c11}. */
3725 uXcptBitmap &= ~HMVMX_REAL_MODE_XCPT_MASK;
3726 if (fInterceptMF)
3727 uXcptBitmap |= RT_BIT(X86_XCPT_MF);
3728 }
3729
3730 /* Additional intercepts for debugging, define these yourself explicitly. */
3731#ifdef HMVMX_ALWAYS_TRAP_ALL_XCPTS
3732 uXcptBitmap |= 0
3733 | RT_BIT(X86_XCPT_BP)
3734 | RT_BIT(X86_XCPT_DE)
3735 | RT_BIT(X86_XCPT_NM)
3736 | RT_BIT(X86_XCPT_TS)
3737 | RT_BIT(X86_XCPT_UD)
3738 | RT_BIT(X86_XCPT_NP)
3739 | RT_BIT(X86_XCPT_SS)
3740 | RT_BIT(X86_XCPT_GP)
3741 | RT_BIT(X86_XCPT_PF)
3742 | RT_BIT(X86_XCPT_MF)
3743 ;
3744#elif defined(HMVMX_ALWAYS_TRAP_PF)
3745 uXcptBitmap |= RT_BIT(X86_XCPT_PF);
3746#endif
3747 if (uXcptBitmap != pVCpu->hm.s.vmx.u32XcptBitmap)
3748 {
3749 pVCpu->hm.s.vmx.u32XcptBitmap = uXcptBitmap;
3750 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_XCPT_INTERCEPTS);
3751 }
3752 Assert(pVM->hm.s.fNestedPaging || (pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT(X86_XCPT_PF)));
3753
3754 /*
3755 * Set/clear the CR0 specific bits along with their exceptions (PE, PG, CD, NW).
3756 */
3757 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3758 uint32_t fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
3759 if (pVM->hm.s.vmx.fUnrestrictedGuest) /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG). */
3760 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
3761 else
3762 Assert((fSetCR0 & (X86_CR0_PE | X86_CR0_PG)) == (X86_CR0_PE | X86_CR0_PG));
3763
3764 uGuestCR0 |= fSetCR0;
3765 uGuestCR0 &= fZapCR0;
3766 uGuestCR0 &= ~(X86_CR0_CD | X86_CR0_NW); /* Always enable caching. */
3767
3768 /*
3769 * CR0 is shared between host and guest along with a CR0 read shadow. Therefore, certain bits must not be changed
3770 * by the guest because VT-x ignores saving/restoring them (namely CD, ET, NW) and for certain other bits
3771 * we want to be notified immediately of guest CR0 changes (e.g. PG to update our shadow page tables).
3772 */
3773 uint32_t uCR0Mask = X86_CR0_PE
3774 | X86_CR0_NE
3775 | (pVM->hm.s.fNestedPaging ? 0 : X86_CR0_WP)
3776 | X86_CR0_PG
3777 | X86_CR0_ET /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.ET */
3778 | X86_CR0_CD /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.CD */
3779 | X86_CR0_NW; /* Bit ignored on VM-entry and VM-exit. Don't let the guest modify the host CR0.NW */
3780
3781 /** @todo Avoid intercepting CR0.PE with unrestricted guests. Fix PGM
3782 * enmGuestMode to be in-sync with the current mode. See @bugref{6398}
3783 * and @bugref{6944}. */
3784#if 0
3785 if (pVM->hm.s.vmx.fUnrestrictedGuest)
3786 uCr0Mask &= ~X86_CR0_PE;
3787#endif
3788 /*
3789 * Finally, update VMCS fields with the CR0 values.
3790 */
3791 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR0, uGuestCR0);
3792 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, uShadowCR0);
3793 if (uCR0Mask != pVCpu->hm.s.vmx.u32CR0Mask)
3794 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, uCR0Mask);
3795 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
3796 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
3797 AssertRCReturn(rc, rc);
3798
3799 /* Update our caches. */
3800 pVCpu->hm.s.vmx.u32CR0Mask = uCR0Mask;
3801 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
3802
3803 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR0);
3804
3805 Log4Func(("uCr0Mask=%#RX32 uShadowCR0=%#RX32 uGuestCR0=%#RX32 (fSetCR0=%#RX32 fZapCR0=%#RX32\n", uCR0Mask, uShadowCR0,
3806 uGuestCR0, fSetCR0, fZapCR0));
3807 }
3808
3809 return VINF_SUCCESS;
3810}
3811
3812
3813/**
3814 * Exports the guest control registers (CR3, CR4) into the guest-state area
3815 * in the VMCS.
3816 *
3817 * @returns VBox strict status code.
3818 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
3819 * without unrestricted guest access and the VMMDev is not presently
3820 * mapped (e.g. EFI32).
3821 *
3822 * @param pVCpu The cross context virtual CPU structure.
3823 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
3824 * out-of-sync. Make sure to update the required fields
3825 * before using them.
3826 *
3827 * @remarks No-long-jump zone!!!
3828 */
3829static VBOXSTRICTRC hmR0VmxExportGuestCR3AndCR4(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
3830{
3831 int rc = VINF_SUCCESS;
3832 PVM pVM = pVCpu->CTX_SUFF(pVM);
3833
3834 /*
3835 * Guest CR2.
3836 * It's always loaded in the assembler code. Nothing to do here.
3837 */
3838
3839 /*
3840 * Guest CR3.
3841 */
3842 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR3)
3843 {
3844 RTGCPHYS GCPhysGuestCR3 = NIL_RTGCPHYS;
3845 if (pVM->hm.s.fNestedPaging)
3846 {
3847 pVCpu->hm.s.vmx.HCPhysEPTP = PGMGetHyperCR3(pVCpu);
3848
3849 /* Validate. See Intel spec. 28.2.2 "EPT Translation Mechanism" and 24.6.11 "Extended-Page-Table Pointer (EPTP)" */
3850 Assert(pVCpu->hm.s.vmx.HCPhysEPTP);
3851 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & UINT64_C(0xfff0000000000000)));
3852 Assert(!(pVCpu->hm.s.vmx.HCPhysEPTP & 0xfff));
3853
3854 /* VMX_EPT_MEMTYPE_WB support is already checked in hmR0VmxSetupTaggedTlb(). */
3855 pVCpu->hm.s.vmx.HCPhysEPTP |= VMX_EPT_MEMTYPE_WB
3856 | (VMX_EPT_PAGE_WALK_LENGTH_DEFAULT << VMX_EPT_PAGE_WALK_LENGTH_SHIFT);
3857
3858 /* Validate. See Intel spec. 26.2.1 "Checks on VMX Controls" */
3859 AssertMsg( ((pVCpu->hm.s.vmx.HCPhysEPTP >> 3) & 0x07) == 3 /* Bits 3:5 (EPT page walk length - 1) must be 3. */
3860 && ((pVCpu->hm.s.vmx.HCPhysEPTP >> 7) & 0x1f) == 0, /* Bits 7:11 MBZ. */
3861 ("EPTP %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3862 AssertMsg( !((pVCpu->hm.s.vmx.HCPhysEPTP >> 6) & 0x01) /* Bit 6 (EPT accessed & dirty bit). */
3863 || (pVM->hm.s.vmx.Msrs.u64EptVpidCaps & MSR_IA32_VMX_EPT_VPID_CAP_EPT_ACCESS_DIRTY),
3864 ("EPTP accessed/dirty bit not supported by CPU but set %#RX64\n", pVCpu->hm.s.vmx.HCPhysEPTP));
3865
3866 rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, pVCpu->hm.s.vmx.HCPhysEPTP);
3867 AssertRCReturn(rc, rc);
3868
3869 if ( pVM->hm.s.vmx.fUnrestrictedGuest
3870 || CPUMIsGuestPagingEnabledEx(pMixedCtx))
3871 {
3872 /* If the guest is in PAE mode, pass the PDPEs to VT-x using the VMCS fields. */
3873 if (CPUMIsGuestInPAEModeEx(pMixedCtx))
3874 {
3875 rc = PGMGstGetPaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
3876 AssertRCReturn(rc, rc);
3877 rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, pVCpu->hm.s.aPdpes[0].u);
3878 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, pVCpu->hm.s.aPdpes[1].u);
3879 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, pVCpu->hm.s.aPdpes[2].u);
3880 rc |= VMXWriteVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, pVCpu->hm.s.aPdpes[3].u);
3881 AssertRCReturn(rc, rc);
3882 }
3883
3884 /*
3885 * The guest's view of its CR3 is unblemished with Nested Paging when the
3886 * guest is using paging or we have unrestricted guest execution to handle
3887 * the guest when it's not using paging.
3888 */
3889 GCPhysGuestCR3 = pMixedCtx->cr3;
3890 }
3891 else
3892 {
3893 /*
3894 * The guest is not using paging, but the CPU (VT-x) has to. While the guest
3895 * thinks it accesses physical memory directly, we use our identity-mapped
3896 * page table to map guest-linear to guest-physical addresses. EPT takes care
3897 * of translating it to host-physical addresses.
3898 */
3899 RTGCPHYS GCPhys;
3900 Assert(pVM->hm.s.vmx.pNonPagingModeEPTPageTable);
3901
3902 /* We obtain it here every time as the guest could have relocated this PCI region. */
3903 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pNonPagingModeEPTPageTable, &GCPhys);
3904 if (RT_SUCCESS(rc))
3905 { /* likely */ }
3906 else if (rc == VERR_PDM_DEV_HEAP_R3_TO_GCPHYS)
3907 {
3908 Log4Func(("VERR_PDM_DEV_HEAP_R3_TO_GCPHYS -> VINF_EM_RESCHEDULE_REM\n"));
3909 return VINF_EM_RESCHEDULE_REM; /* We cannot execute now, switch to REM/IEM till the guest maps in VMMDev. */
3910 }
3911 else
3912 AssertMsgFailedReturn(("%Rrc\n", rc), rc);
3913
3914 GCPhysGuestCR3 = GCPhys;
3915 }
3916
3917 Log4Func(("uGuestCR3=%#RGp (GstN)\n", GCPhysGuestCR3));
3918 rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_CR3, GCPhysGuestCR3);
3919 AssertRCReturn(rc, rc);
3920 }
3921 else
3922 {
3923 /* Non-nested paging case, just use the hypervisor's CR3. */
3924 RTHCPHYS HCPhysGuestCR3 = PGMGetHyperCR3(pVCpu);
3925
3926 Log4Func(("uGuestCR3=%#RHv (HstN)\n", HCPhysGuestCR3));
3927 rc = VMXWriteVmcsHstN(VMX_VMCS_GUEST_CR3, HCPhysGuestCR3);
3928 AssertRCReturn(rc, rc);
3929 }
3930
3931 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR3);
3932 }
3933
3934 /*
3935 * Guest CR4.
3936 * ASSUMES this is done everytime we get in from ring-3! (XCR0)
3937 */
3938 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_CR4)
3939 {
3940 Assert(!RT_HI_U32(pMixedCtx->cr4));
3941 uint32_t uGuestCR4 = pMixedCtx->cr4;
3942 uint32_t const uShadowCR4 = pMixedCtx->cr4;
3943
3944 /*
3945 * Setup VT-x's view of the guest CR4.
3946 *
3947 * If we're emulating real-mode using virtual-8086 mode, we want to redirect software
3948 * interrupts to the 8086 program interrupt handler. Clear the VME bit (the interrupt
3949 * redirection bitmap is already all 0, see hmR3InitFinalizeR0())
3950 *
3951 * See Intel spec. 20.2 "Software Interrupt Handling Methods While in Virtual-8086 Mode".
3952 */
3953 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
3954 {
3955 Assert(pVM->hm.s.vmx.pRealModeTSS);
3956 Assert(PDMVmmDevHeapIsEnabled(pVM));
3957 uGuestCR4 &= ~X86_CR4_VME;
3958 }
3959
3960 if (pVM->hm.s.fNestedPaging)
3961 {
3962 if ( !CPUMIsGuestPagingEnabledEx(pMixedCtx)
3963 && !pVM->hm.s.vmx.fUnrestrictedGuest)
3964 {
3965 /* We use 4 MB pages in our identity mapping page table when the guest doesn't have paging. */
3966 uGuestCR4 |= X86_CR4_PSE;
3967 /* Our identity mapping is a 32-bit page directory. */
3968 uGuestCR4 &= ~X86_CR4_PAE;
3969 }
3970 /* else use guest CR4.*/
3971 }
3972 else
3973 {
3974 /*
3975 * The shadow paging modes and guest paging modes are different, the shadow is in accordance with the host
3976 * paging mode and thus we need to adjust VT-x's view of CR4 depending on our shadow page tables.
3977 */
3978 switch (pVCpu->hm.s.enmShadowMode)
3979 {
3980 case PGMMODE_REAL: /* Real-mode. */
3981 case PGMMODE_PROTECTED: /* Protected mode without paging. */
3982 case PGMMODE_32_BIT: /* 32-bit paging. */
3983 {
3984 uGuestCR4 &= ~X86_CR4_PAE;
3985 break;
3986 }
3987
3988 case PGMMODE_PAE: /* PAE paging. */
3989 case PGMMODE_PAE_NX: /* PAE paging with NX. */
3990 {
3991 uGuestCR4 |= X86_CR4_PAE;
3992 break;
3993 }
3994
3995 case PGMMODE_AMD64: /* 64-bit AMD paging (long mode). */
3996 case PGMMODE_AMD64_NX: /* 64-bit AMD paging (long mode) with NX enabled. */
3997#ifdef VBOX_ENABLE_64_BITS_GUESTS
3998 break;
3999#endif
4000 default:
4001 AssertFailed();
4002 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4003 }
4004 }
4005
4006 /* We need to set and clear the CR4 specific bits here (mainly the X86_CR4_VMXE bit). */
4007 uint64_t const fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4008 uint64_t const fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
4009 uGuestCR4 |= fSetCR4;
4010 uGuestCR4 &= fZapCR4;
4011
4012 /* Setup CR4 mask. CR4 flags owned by the host, if the guest attempts to change them,
4013 that would cause a VM-exit. */
4014 uint32_t u32CR4Mask = X86_CR4_VME
4015 | X86_CR4_PAE
4016 | X86_CR4_PGE
4017 | X86_CR4_PSE
4018 | X86_CR4_VMXE;
4019 if (pVM->cpum.ro.HostFeatures.fXSaveRstor)
4020 u32CR4Mask |= X86_CR4_OSXSAVE;
4021 if (pVM->cpum.ro.GuestFeatures.fPcid)
4022 u32CR4Mask |= X86_CR4_PCIDE;
4023
4024 /* Write VT-x's view of the guest CR4, the CR4 modify mask and the read-only CR4 shadow
4025 into the VMCS and update our cache. */
4026 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_CR4, uGuestCR4);
4027 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, uShadowCR4);
4028 if (pVCpu->hm.s.vmx.u32CR4Mask != u32CR4Mask)
4029 rc |= VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, u32CR4Mask);
4030 AssertRCReturn(rc, rc);
4031 pVCpu->hm.s.vmx.u32CR4Mask = u32CR4Mask;
4032
4033 /* Whether to save/load/restore XCR0 during world switch depends on CR4.OSXSAVE and host+guest XCR0. */
4034 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
4035
4036 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_CR4);
4037
4038 Log4Func(("uGuestCR4=%#RX32 uShadowCR4=%#RX32 (fSetCR4=%#RX32 fZapCR4=%#RX32)\n", uGuestCR4, uShadowCR4, fSetCR4,
4039 fZapCR4));
4040 }
4041 return rc;
4042}
4043
4044
4045/**
4046 * Exports the guest debug registers into the guest-state area in the VMCS.
4047 * The guest debug bits are partially shared with the host (e.g. DR6, DR0-3).
4048 *
4049 * This also sets up whether \#DB and MOV DRx accesses cause VM-exits.
4050 *
4051 * @returns VBox status code.
4052 * @param pVCpu The cross context virtual CPU structure.
4053 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4054 * out-of-sync. Make sure to update the required fields
4055 * before using them.
4056 *
4057 * @remarks No-long-jump zone!!!
4058 */
4059static int hmR0VmxExportSharedDebugState(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
4060{
4061 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
4062
4063#ifdef VBOX_STRICT
4064 /* Validate. Intel spec. 26.3.1.1 "Checks on Guest Controls Registers, Debug Registers, MSRs" */
4065 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
4066 {
4067 /* Validate. Intel spec. 17.2 "Debug Registers", recompiler paranoia checks. */
4068 Assert((pMixedCtx->dr[7] & (X86_DR7_MBZ_MASK | X86_DR7_RAZ_MASK)) == 0); /* Bits 63:32, 15, 14, 12, 11 are reserved. */
4069 Assert((pMixedCtx->dr[7] & X86_DR7_RA1_MASK) == X86_DR7_RA1_MASK); /* Bit 10 is reserved (RA1). */
4070 }
4071#endif
4072
4073 bool fSteppingDB = false;
4074 bool fInterceptMovDRx = false;
4075 uint32_t uProcCtls = pVCpu->hm.s.vmx.u32ProcCtls;
4076 if (pVCpu->hm.s.fSingleInstruction)
4077 {
4078 /* If the CPU supports the monitor trap flag, use it for single stepping in DBGF and avoid intercepting #DB. */
4079 PVM pVM = pVCpu->CTX_SUFF(pVM);
4080 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG)
4081 {
4082 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
4083 Assert(fSteppingDB == false);
4084 }
4085 else
4086 {
4087 pMixedCtx->eflags.u32 |= X86_EFL_TF;
4088 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_RFLAGS;
4089 pVCpu->hm.s.fClearTrapFlag = true;
4090 fSteppingDB = true;
4091 }
4092 }
4093
4094 uint32_t uGuestDR7;
4095 if ( fSteppingDB
4096 || (CPUMGetHyperDR7(pVCpu) & X86_DR7_ENABLED_MASK))
4097 {
4098 /*
4099 * Use the combined guest and host DRx values found in the hypervisor register set
4100 * because the debugger has breakpoints active or someone is single stepping on the
4101 * host side without a monitor trap flag.
4102 *
4103 * Note! DBGF expects a clean DR6 state before executing guest code.
4104 */
4105#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4106 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4107 && !CPUMIsHyperDebugStateActivePending(pVCpu))
4108 {
4109 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4110 Assert(CPUMIsHyperDebugStateActivePending(pVCpu));
4111 Assert(!CPUMIsGuestDebugStateActivePending(pVCpu));
4112 }
4113 else
4114#endif
4115 if (!CPUMIsHyperDebugStateActive(pVCpu))
4116 {
4117 CPUMR0LoadHyperDebugState(pVCpu, true /* include DR6 */);
4118 Assert(CPUMIsHyperDebugStateActive(pVCpu));
4119 Assert(!CPUMIsGuestDebugStateActive(pVCpu));
4120 }
4121
4122 /* Update DR7 with the hypervisor value (other DRx registers are handled by CPUM one way or another). */
4123 uGuestDR7 = (uint32_t)CPUMGetHyperDR7(pVCpu);
4124 pVCpu->hm.s.fUsingHyperDR7 = true;
4125 fInterceptMovDRx = true;
4126 }
4127 else
4128 {
4129 /*
4130 * If the guest has enabled debug registers, we need to load them prior to
4131 * executing guest code so they'll trigger at the right time.
4132 */
4133 if (pMixedCtx->dr[7] & (X86_DR7_ENABLED_MASK | X86_DR7_GD))
4134 {
4135#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4136 if ( CPUMIsGuestInLongModeEx(pMixedCtx)
4137 && !CPUMIsGuestDebugStateActivePending(pVCpu))
4138 {
4139 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4140 Assert(CPUMIsGuestDebugStateActivePending(pVCpu));
4141 Assert(!CPUMIsHyperDebugStateActivePending(pVCpu));
4142 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4143 }
4144 else
4145#endif
4146 if (!CPUMIsGuestDebugStateActive(pVCpu))
4147 {
4148 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
4149 Assert(CPUMIsGuestDebugStateActive(pVCpu));
4150 Assert(!CPUMIsHyperDebugStateActive(pVCpu));
4151 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxArmed);
4152 }
4153 Assert(!fInterceptMovDRx);
4154 }
4155 /*
4156 * If no debugging enabled, we'll lazy load DR0-3. Unlike on AMD-V, we
4157 * must intercept #DB in order to maintain a correct DR6 guest value, and
4158 * because we need to intercept it to prevent nested #DBs from hanging the
4159 * CPU, we end up always having to intercept it. See hmR0VmxInitXcptBitmap.
4160 */
4161#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
4162 else if ( !CPUMIsGuestDebugStateActivePending(pVCpu)
4163 && !CPUMIsGuestDebugStateActive(pVCpu))
4164#else
4165 else if (!CPUMIsGuestDebugStateActive(pVCpu))
4166#endif
4167 {
4168 fInterceptMovDRx = true;
4169 }
4170
4171 /* Update DR7 with the actual guest value. */
4172 uGuestDR7 = pMixedCtx->dr[7];
4173 pVCpu->hm.s.fUsingHyperDR7 = false;
4174 }
4175
4176 if (fInterceptMovDRx)
4177 uProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4178 else
4179 uProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
4180
4181 /*
4182 * Update the processor-based VM-execution controls with the MOV-DRx intercepts and the
4183 * monitor-trap flag and update our cache.
4184 */
4185 if (uProcCtls != pVCpu->hm.s.vmx.u32ProcCtls)
4186 {
4187 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, uProcCtls);
4188 AssertRCReturn(rc2, rc2);
4189 pVCpu->hm.s.vmx.u32ProcCtls = uProcCtls;
4190 }
4191
4192 /*
4193 * Update guest DR7.
4194 */
4195 int rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, uGuestDR7);
4196 AssertRCReturn(rc, rc);
4197
4198 return VINF_SUCCESS;
4199}
4200
4201
4202#ifdef VBOX_STRICT
4203/**
4204 * Strict function to validate segment registers.
4205 *
4206 * @param pVCpu The cross context virtual CPU structure.
4207 * @param pCtx Pointer to the guest-CPU context.
4208 *
4209 * @remarks Will import guest CR0 on strict builds during validation of
4210 * segments.
4211 */
4212static void hmR0VmxValidateSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pCtx)
4213{
4214 /*
4215 * Validate segment registers. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
4216 *
4217 * The reason we check for attribute value 0 in this function and not just the unusable bit is
4218 * because hmR0VmxWriteSegmentReg() only updates the VMCS' copy of the value with the unusable bit
4219 * and doesn't change the guest-context value.
4220 */
4221 PVM pVM = pVCpu->CTX_SUFF(pVM);
4222 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
4223 if ( !pVM->hm.s.vmx.fUnrestrictedGuest
4224 && ( !CPUMIsGuestInRealModeEx(pCtx)
4225 && !CPUMIsGuestInV86ModeEx(pCtx)))
4226 {
4227 /* Protected mode checks */
4228 /* CS */
4229 Assert(pCtx->cs.Attr.n.u1Present);
4230 Assert(!(pCtx->cs.Attr.u & 0xf00));
4231 Assert(!(pCtx->cs.Attr.u & 0xfffe0000));
4232 Assert( (pCtx->cs.u32Limit & 0xfff) == 0xfff
4233 || !(pCtx->cs.Attr.n.u1Granularity));
4234 Assert( !(pCtx->cs.u32Limit & 0xfff00000)
4235 || (pCtx->cs.Attr.n.u1Granularity));
4236 /* CS cannot be loaded with NULL in protected mode. */
4237 Assert(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE)); /** @todo is this really true even for 64-bit CS? */
4238 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
4239 Assert(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl);
4240 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
4241 Assert(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl);
4242 else
4243 AssertMsgFailed(("Invalid CS Type %#x\n", pCtx->cs.Attr.n.u2Dpl));
4244 /* SS */
4245 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4246 Assert(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL));
4247 if ( !(pCtx->cr0 & X86_CR0_PE)
4248 || pCtx->cs.Attr.n.u4Type == 3)
4249 {
4250 Assert(!pCtx->ss.Attr.n.u2Dpl);
4251 }
4252 if (pCtx->ss.Attr.u && !(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
4253 {
4254 Assert((pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL));
4255 Assert(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7);
4256 Assert(pCtx->ss.Attr.n.u1Present);
4257 Assert(!(pCtx->ss.Attr.u & 0xf00));
4258 Assert(!(pCtx->ss.Attr.u & 0xfffe0000));
4259 Assert( (pCtx->ss.u32Limit & 0xfff) == 0xfff
4260 || !(pCtx->ss.Attr.n.u1Granularity));
4261 Assert( !(pCtx->ss.u32Limit & 0xfff00000)
4262 || (pCtx->ss.Attr.n.u1Granularity));
4263 }
4264 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
4265 if (pCtx->ds.Attr.u && !(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
4266 {
4267 Assert(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4268 Assert(pCtx->ds.Attr.n.u1Present);
4269 Assert(pCtx->ds.Attr.n.u4Type > 11 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL));
4270 Assert(!(pCtx->ds.Attr.u & 0xf00));
4271 Assert(!(pCtx->ds.Attr.u & 0xfffe0000));
4272 Assert( (pCtx->ds.u32Limit & 0xfff) == 0xfff
4273 || !(pCtx->ds.Attr.n.u1Granularity));
4274 Assert( !(pCtx->ds.u32Limit & 0xfff00000)
4275 || (pCtx->ds.Attr.n.u1Granularity));
4276 Assert( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4277 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ));
4278 }
4279 if (pCtx->es.Attr.u && !(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
4280 {
4281 Assert(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4282 Assert(pCtx->es.Attr.n.u1Present);
4283 Assert(pCtx->es.Attr.n.u4Type > 11 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL));
4284 Assert(!(pCtx->es.Attr.u & 0xf00));
4285 Assert(!(pCtx->es.Attr.u & 0xfffe0000));
4286 Assert( (pCtx->es.u32Limit & 0xfff) == 0xfff
4287 || !(pCtx->es.Attr.n.u1Granularity));
4288 Assert( !(pCtx->es.u32Limit & 0xfff00000)
4289 || (pCtx->es.Attr.n.u1Granularity));
4290 Assert( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4291 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ));
4292 }
4293 if (pCtx->fs.Attr.u && !(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
4294 {
4295 Assert(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4296 Assert(pCtx->fs.Attr.n.u1Present);
4297 Assert(pCtx->fs.Attr.n.u4Type > 11 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL));
4298 Assert(!(pCtx->fs.Attr.u & 0xf00));
4299 Assert(!(pCtx->fs.Attr.u & 0xfffe0000));
4300 Assert( (pCtx->fs.u32Limit & 0xfff) == 0xfff
4301 || !(pCtx->fs.Attr.n.u1Granularity));
4302 Assert( !(pCtx->fs.u32Limit & 0xfff00000)
4303 || (pCtx->fs.Attr.n.u1Granularity));
4304 Assert( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4305 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4306 }
4307 if (pCtx->gs.Attr.u && !(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
4308 {
4309 Assert(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED);
4310 Assert(pCtx->gs.Attr.n.u1Present);
4311 Assert(pCtx->gs.Attr.n.u4Type > 11 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL));
4312 Assert(!(pCtx->gs.Attr.u & 0xf00));
4313 Assert(!(pCtx->gs.Attr.u & 0xfffe0000));
4314 Assert( (pCtx->gs.u32Limit & 0xfff) == 0xfff
4315 || !(pCtx->gs.Attr.n.u1Granularity));
4316 Assert( !(pCtx->gs.u32Limit & 0xfff00000)
4317 || (pCtx->gs.Attr.n.u1Granularity));
4318 Assert( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
4319 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ));
4320 }
4321 /* 64-bit capable CPUs. */
4322# if HC_ARCH_BITS == 64
4323 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4324 Assert(!pCtx->ss.Attr.u || !RT_HI_U32(pCtx->ss.u64Base));
4325 Assert(!pCtx->ds.Attr.u || !RT_HI_U32(pCtx->ds.u64Base));
4326 Assert(!pCtx->es.Attr.u || !RT_HI_U32(pCtx->es.u64Base));
4327# endif
4328 }
4329 else if ( CPUMIsGuestInV86ModeEx(pCtx)
4330 || ( CPUMIsGuestInRealModeEx(pCtx)
4331 && !pVM->hm.s.vmx.fUnrestrictedGuest))
4332 {
4333 /* Real and v86 mode checks. */
4334 /* hmR0VmxWriteSegmentReg() writes the modified in VMCS. We want what we're feeding to VT-x. */
4335 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
4336 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4337 {
4338 u32CSAttr = 0xf3; u32SSAttr = 0xf3; u32DSAttr = 0xf3; u32ESAttr = 0xf3; u32FSAttr = 0xf3; u32GSAttr = 0xf3;
4339 }
4340 else
4341 {
4342 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u; u32DSAttr = pCtx->ds.Attr.u;
4343 u32ESAttr = pCtx->es.Attr.u; u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
4344 }
4345
4346 /* CS */
4347 AssertMsg((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), ("CS base %#x %#x\n", pCtx->cs.u64Base, pCtx->cs.Sel));
4348 Assert(pCtx->cs.u32Limit == 0xffff);
4349 Assert(u32CSAttr == 0xf3);
4350 /* SS */
4351 Assert(pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4);
4352 Assert(pCtx->ss.u32Limit == 0xffff);
4353 Assert(u32SSAttr == 0xf3);
4354 /* DS */
4355 Assert(pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4);
4356 Assert(pCtx->ds.u32Limit == 0xffff);
4357 Assert(u32DSAttr == 0xf3);
4358 /* ES */
4359 Assert(pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4);
4360 Assert(pCtx->es.u32Limit == 0xffff);
4361 Assert(u32ESAttr == 0xf3);
4362 /* FS */
4363 Assert(pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4);
4364 Assert(pCtx->fs.u32Limit == 0xffff);
4365 Assert(u32FSAttr == 0xf3);
4366 /* GS */
4367 Assert(pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4);
4368 Assert(pCtx->gs.u32Limit == 0xffff);
4369 Assert(u32GSAttr == 0xf3);
4370 /* 64-bit capable CPUs. */
4371# if HC_ARCH_BITS == 64
4372 Assert(!RT_HI_U32(pCtx->cs.u64Base));
4373 Assert(!u32SSAttr || !RT_HI_U32(pCtx->ss.u64Base));
4374 Assert(!u32DSAttr || !RT_HI_U32(pCtx->ds.u64Base));
4375 Assert(!u32ESAttr || !RT_HI_U32(pCtx->es.u64Base));
4376# endif
4377 }
4378}
4379#endif /* VBOX_STRICT */
4380
4381
4382/**
4383 * Writes a guest segment register into the guest-state area in the VMCS.
4384 *
4385 * @returns VBox status code.
4386 * @param pVCpu The cross context virtual CPU structure.
4387 * @param idxSel Index of the selector in the VMCS.
4388 * @param idxLimit Index of the segment limit in the VMCS.
4389 * @param idxBase Index of the segment base in the VMCS.
4390 * @param idxAccess Index of the access rights of the segment in the VMCS.
4391 * @param pSelReg Pointer to the segment selector.
4392 *
4393 * @remarks No-long-jump zone!!!
4394 */
4395static int hmR0VmxWriteSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase,
4396 uint32_t idxAccess, PCCPUMSELREG pSelReg)
4397{
4398 int rc = VMXWriteVmcs32(idxSel, pSelReg->Sel); /* 16-bit guest selector field. */
4399 rc |= VMXWriteVmcs32(idxLimit, pSelReg->u32Limit); /* 32-bit guest segment limit field. */
4400 rc |= VMXWriteVmcsGstN(idxBase, pSelReg->u64Base); /* Natural width guest segment base field.*/
4401 AssertRCReturn(rc, rc);
4402
4403 uint32_t u32Access = pSelReg->Attr.u;
4404 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4405 {
4406 /* VT-x requires our real-using-v86 mode hack to override the segment access-right bits. */
4407 u32Access = 0xf3;
4408 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
4409 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
4410 }
4411 else
4412 {
4413 /*
4414 * The way to differentiate between whether this is really a null selector or was just
4415 * a selector loaded with 0 in real-mode is using the segment attributes. A selector
4416 * loaded in real-mode with the value 0 is valid and usable in protected-mode and we
4417 * should -not- mark it as an unusable segment. Both the recompiler & VT-x ensures
4418 * NULL selectors loaded in protected-mode have their attribute as 0.
4419 */
4420 if (!u32Access)
4421 u32Access = X86DESCATTR_UNUSABLE;
4422 }
4423
4424 /* Validate segment access rights. Refer to Intel spec. "26.3.1.2 Checks on Guest Segment Registers". */
4425 AssertMsg((u32Access & X86DESCATTR_UNUSABLE) || (u32Access & X86_SEL_TYPE_ACCESSED),
4426 ("Access bit not set for usable segment. idx=%#x sel=%#x attr %#x\n", idxBase, pSelReg, pSelReg->Attr.u));
4427
4428 rc = VMXWriteVmcs32(idxAccess, u32Access); /* 32-bit guest segment access-rights field. */
4429 AssertRCReturn(rc, rc);
4430 return rc;
4431}
4432
4433
4434/**
4435 * Exports the guest segment registers, GDTR, IDTR, LDTR, (TR, FS and GS bases)
4436 * into the guest-state area in the VMCS.
4437 *
4438 * @returns VBox status code.
4439 * @param pVCpu The cross context virtual CPU structure.
4440 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4441 * out-of-sync. Make sure to update the required fields
4442 * before using them.
4443 *
4444 * @remarks Will import guest CR0 on strict builds during validation of
4445 * segments.
4446 * @remarks No-long-jump zone!!!
4447 */
4448static int hmR0VmxExportGuestSegmentRegs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4449{
4450 int rc = VERR_INTERNAL_ERROR_5;
4451 PVM pVM = pVCpu->CTX_SUFF(pVM);
4452
4453 /*
4454 * Guest Segment registers: CS, SS, DS, ES, FS, GS.
4455 */
4456 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SREG_MASK)
4457 {
4458 /* Save the segment attributes for real-on-v86 mode hack, so we can restore them on VM-exit. */
4459 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4460 {
4461 pVCpu->hm.s.vmx.RealMode.AttrCS.u = pMixedCtx->cs.Attr.u;
4462 pVCpu->hm.s.vmx.RealMode.AttrSS.u = pMixedCtx->ss.Attr.u;
4463 pVCpu->hm.s.vmx.RealMode.AttrDS.u = pMixedCtx->ds.Attr.u;
4464 pVCpu->hm.s.vmx.RealMode.AttrES.u = pMixedCtx->es.Attr.u;
4465 pVCpu->hm.s.vmx.RealMode.AttrFS.u = pMixedCtx->fs.Attr.u;
4466 pVCpu->hm.s.vmx.RealMode.AttrGS.u = pMixedCtx->gs.Attr.u;
4467 }
4468
4469#ifdef VBOX_WITH_REM
4470 if (!pVM->hm.s.vmx.fUnrestrictedGuest)
4471 {
4472 Assert(pVM->hm.s.vmx.pRealModeTSS);
4473 AssertCompile(PGMMODE_REAL < PGMMODE_PROTECTED);
4474 if ( pVCpu->hm.s.vmx.fWasInRealMode
4475 && PGMGetGuestMode(pVCpu) >= PGMMODE_PROTECTED)
4476 {
4477 /* Signal that the recompiler must flush its code-cache as the guest -may- rewrite code it will later execute
4478 in real-mode (e.g. OpenBSD 4.0) */
4479 REMFlushTBs(pVM);
4480 Log4Func(("Switch to protected mode detected!\n"));
4481 pVCpu->hm.s.vmx.fWasInRealMode = false;
4482 }
4483 }
4484#endif
4485 rc = hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_CS_SEL, VMX_VMCS32_GUEST_CS_LIMIT, VMX_VMCS_GUEST_CS_BASE,
4486 VMX_VMCS32_GUEST_CS_ACCESS_RIGHTS, &pMixedCtx->cs);
4487 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_SS_SEL, VMX_VMCS32_GUEST_SS_LIMIT, VMX_VMCS_GUEST_SS_BASE,
4488 VMX_VMCS32_GUEST_SS_ACCESS_RIGHTS, &pMixedCtx->ss);
4489 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_DS_SEL, VMX_VMCS32_GUEST_DS_LIMIT, VMX_VMCS_GUEST_DS_BASE,
4490 VMX_VMCS32_GUEST_DS_ACCESS_RIGHTS, &pMixedCtx->ds);
4491 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_ES_SEL, VMX_VMCS32_GUEST_ES_LIMIT, VMX_VMCS_GUEST_ES_BASE,
4492 VMX_VMCS32_GUEST_ES_ACCESS_RIGHTS, &pMixedCtx->es);
4493 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_FS_SEL, VMX_VMCS32_GUEST_FS_LIMIT, VMX_VMCS_GUEST_FS_BASE,
4494 VMX_VMCS32_GUEST_FS_ACCESS_RIGHTS, &pMixedCtx->fs);
4495 rc |= hmR0VmxWriteSegmentReg(pVCpu, VMX_VMCS16_GUEST_GS_SEL, VMX_VMCS32_GUEST_GS_LIMIT, VMX_VMCS_GUEST_GS_BASE,
4496 VMX_VMCS32_GUEST_GS_ACCESS_RIGHTS, &pMixedCtx->gs);
4497 AssertRCReturn(rc, rc);
4498
4499#ifdef VBOX_STRICT
4500 hmR0VmxValidateSegmentRegs(pVCpu, pMixedCtx);
4501#endif
4502
4503 /* Update the exit history entry with the correct CS.BASE + RIP. */
4504 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_RIP)
4505 EMR0HistoryUpdatePC(pVCpu, pMixedCtx->cs.u64Base + pMixedCtx->rip, true);
4506
4507 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SREG_MASK);
4508 Log4Func(("CS=%#RX16 Base=%#RX64 Limit=%#RX32 Attr=%#RX32\n", pMixedCtx->cs.Sel, pMixedCtx->cs.u64Base,
4509 pMixedCtx->cs.u32Limit, pMixedCtx->cs.Attr.u));
4510 }
4511
4512 /*
4513 * Guest TR.
4514 */
4515 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_TR)
4516 {
4517 /*
4518 * Real-mode emulation using virtual-8086 mode with CR4.VME. Interrupt redirection is
4519 * achieved using the interrupt redirection bitmap (all bits cleared to let the guest
4520 * handle INT-n's) in the TSS. See hmR3InitFinalizeR0() to see how pRealModeTSS is setup.
4521 */
4522 uint16_t u16Sel = 0;
4523 uint32_t u32Limit = 0;
4524 uint64_t u64Base = 0;
4525 uint32_t u32AccessRights = 0;
4526
4527 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
4528 {
4529 u16Sel = pMixedCtx->tr.Sel;
4530 u32Limit = pMixedCtx->tr.u32Limit;
4531 u64Base = pMixedCtx->tr.u64Base;
4532 u32AccessRights = pMixedCtx->tr.Attr.u;
4533 }
4534 else
4535 {
4536 Assert(pVM->hm.s.vmx.pRealModeTSS);
4537 Assert(PDMVmmDevHeapIsEnabled(pVM)); /* Guaranteed by HMR3CanExecuteGuest() -XXX- what about inner loop changes? */
4538
4539 /* We obtain it here every time as PCI regions could be reconfigured in the guest, changing the VMMDev base. */
4540 RTGCPHYS GCPhys;
4541 rc = PDMVmmDevHeapR3ToGCPhys(pVM, pVM->hm.s.vmx.pRealModeTSS, &GCPhys);
4542 AssertRCReturn(rc, rc);
4543
4544 X86DESCATTR DescAttr;
4545 DescAttr.u = 0;
4546 DescAttr.n.u1Present = 1;
4547 DescAttr.n.u4Type = X86_SEL_TYPE_SYS_386_TSS_BUSY;
4548
4549 u16Sel = 0;
4550 u32Limit = HM_VTX_TSS_SIZE;
4551 u64Base = GCPhys; /* in real-mode phys = virt. */
4552 u32AccessRights = DescAttr.u;
4553 }
4554
4555 /* Validate. */
4556 Assert(!(u16Sel & RT_BIT(2)));
4557 AssertMsg( (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_386_TSS_BUSY
4558 || (u32AccessRights & 0xf) == X86_SEL_TYPE_SYS_286_TSS_BUSY, ("TSS is not busy!? %#x\n", u32AccessRights));
4559 AssertMsg(!(u32AccessRights & X86DESCATTR_UNUSABLE), ("TR unusable bit is not clear!? %#x\n", u32AccessRights));
4560 Assert(!(u32AccessRights & RT_BIT(4))); /* System MBZ.*/
4561 Assert(u32AccessRights & RT_BIT(7)); /* Present MB1.*/
4562 Assert(!(u32AccessRights & 0xf00)); /* 11:8 MBZ. */
4563 Assert(!(u32AccessRights & 0xfffe0000)); /* 31:17 MBZ. */
4564 Assert( (u32Limit & 0xfff) == 0xfff
4565 || !(u32AccessRights & RT_BIT(15))); /* Granularity MBZ. */
4566 Assert( !(pMixedCtx->tr.u32Limit & 0xfff00000)
4567 || (u32AccessRights & RT_BIT(15))); /* Granularity MB1. */
4568
4569 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_TR_SEL, u16Sel);
4570 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_LIMIT, u32Limit);
4571 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_TR_ACCESS_RIGHTS, u32AccessRights);
4572 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_TR_BASE, u64Base);
4573 AssertRCReturn(rc, rc);
4574
4575 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_TR);
4576 Log4Func(("TR base=%#RX64\n", pMixedCtx->tr.u64Base));
4577 }
4578
4579 /*
4580 * Guest GDTR.
4581 */
4582 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_GDTR)
4583 {
4584 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, pMixedCtx->gdtr.cbGdt);
4585 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, pMixedCtx->gdtr.pGdt);
4586 AssertRCReturn(rc, rc);
4587
4588 /* Validate. */
4589 Assert(!(pMixedCtx->gdtr.cbGdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4590
4591 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_GDTR);
4592 Log4Func(("GDTR base=%#RX64\n", pMixedCtx->gdtr.pGdt));
4593 }
4594
4595 /*
4596 * Guest LDTR.
4597 */
4598 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_LDTR)
4599 {
4600 /* The unusable bit is specific to VT-x, if it's a null selector mark it as an unusable segment. */
4601 uint32_t u32Access = 0;
4602 if (!pMixedCtx->ldtr.Attr.u)
4603 u32Access = X86DESCATTR_UNUSABLE;
4604 else
4605 u32Access = pMixedCtx->ldtr.Attr.u;
4606
4607 rc = VMXWriteVmcs32(VMX_VMCS16_GUEST_LDTR_SEL, pMixedCtx->ldtr.Sel);
4608 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_LIMIT, pMixedCtx->ldtr.u32Limit);
4609 rc |= VMXWriteVmcs32(VMX_VMCS32_GUEST_LDTR_ACCESS_RIGHTS, u32Access);
4610 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_LDTR_BASE, pMixedCtx->ldtr.u64Base);
4611 AssertRCReturn(rc, rc);
4612
4613 /* Validate. */
4614 if (!(u32Access & X86DESCATTR_UNUSABLE))
4615 {
4616 Assert(!(pMixedCtx->ldtr.Sel & RT_BIT(2))); /* TI MBZ. */
4617 Assert(pMixedCtx->ldtr.Attr.n.u4Type == 2); /* Type MB2 (LDT). */
4618 Assert(!pMixedCtx->ldtr.Attr.n.u1DescType); /* System MBZ. */
4619 Assert(pMixedCtx->ldtr.Attr.n.u1Present == 1); /* Present MB1. */
4620 Assert(!pMixedCtx->ldtr.Attr.n.u4LimitHigh); /* 11:8 MBZ. */
4621 Assert(!(pMixedCtx->ldtr.Attr.u & 0xfffe0000)); /* 31:17 MBZ. */
4622 Assert( (pMixedCtx->ldtr.u32Limit & 0xfff) == 0xfff
4623 || !pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MBZ. */
4624 Assert( !(pMixedCtx->ldtr.u32Limit & 0xfff00000)
4625 || pMixedCtx->ldtr.Attr.n.u1Granularity); /* Granularity MB1. */
4626 }
4627
4628 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_LDTR);
4629 Log4Func(("LDTR base=%#RX64\n", pMixedCtx->ldtr.u64Base));
4630 }
4631
4632 /*
4633 * Guest IDTR.
4634 */
4635 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_IDTR)
4636 {
4637 rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, pMixedCtx->idtr.cbIdt);
4638 rc |= VMXWriteVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, pMixedCtx->idtr.pIdt);
4639 AssertRCReturn(rc, rc);
4640
4641 /* Validate. */
4642 Assert(!(pMixedCtx->idtr.cbIdt & 0xffff0000)); /* Bits 31:16 MBZ. */
4643
4644 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_IDTR);
4645 Log4Func(("IDTR base=%#RX64\n", pMixedCtx->idtr.pIdt));
4646 }
4647
4648 return VINF_SUCCESS;
4649}
4650
4651
4652/**
4653 * Exports certain guest MSRs into the VM-entry MSR-load and VM-exit MSR-store
4654 * areas.
4655 *
4656 * These MSRs will automatically be loaded to the host CPU on every successful
4657 * VM-entry and stored from the host CPU on every successful VM-exit. This also
4658 * creates/updates MSR slots for the host MSRs. The actual host MSR values are
4659 * -not- updated here for performance reasons. See hmR0VmxExportHostMsrs().
4660 *
4661 * Also exports the guest sysenter MSRs into the guest-state area in the VMCS.
4662 *
4663 * @returns VBox status code.
4664 * @param pVCpu The cross context virtual CPU structure.
4665 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4666 * out-of-sync. Make sure to update the required fields
4667 * before using them.
4668 *
4669 * @remarks No-long-jump zone!!!
4670 */
4671static int hmR0VmxExportGuestMsrs(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4672{
4673 AssertPtr(pVCpu);
4674 AssertPtr(pVCpu->hm.s.vmx.pvGuestMsr);
4675
4676 /*
4677 * MSRs that we use the auto-load/store MSR area in the VMCS.
4678 * For 64-bit hosts, we load/restore them lazily, see hmR0VmxLazyLoadGuestMsrs().
4679 */
4680 PVM pVM = pVCpu->CTX_SUFF(pVM);
4681 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_VMX_GUEST_AUTO_MSRS)
4682 {
4683 if (pVM->hm.s.fAllow64BitGuests)
4684 {
4685#if HC_ARCH_BITS == 32
4686 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_LSTAR, pMixedCtx->msrLSTAR, false, NULL);
4687 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_STAR, pMixedCtx->msrSTAR, false, NULL);
4688 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_SF_MASK, pMixedCtx->msrSFMASK, false, NULL);
4689 rc |= hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_KERNEL_GS_BASE, pMixedCtx->msrKERNELGSBASE, false, NULL);
4690 AssertRCReturn(rc, rc);
4691# ifdef LOG_ENABLED
4692 PCVMXAUTOMSR pMsr = (PCVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
4693 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.cMsrs; i++, pMsr++)
4694 Log4Func(("MSR[%RU32]: u32Msr=%#RX32 u64Value=%#RX64\n", i, pMsr->u32Msr, pMsr->u64Value));
4695# endif
4696#endif
4697 }
4698 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_VMX_GUEST_AUTO_MSRS);
4699 }
4700
4701 /*
4702 * Guest Sysenter MSRs.
4703 * These flags are only set when MSR-bitmaps are not supported by the CPU and we cause
4704 * VM-exits on WRMSRs for these MSRs.
4705 */
4706 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_MSR_MASK)
4707 {
4708 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_CS_MSR)
4709 {
4710 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, pMixedCtx->SysEnter.cs);
4711 AssertRCReturn(rc, rc);
4712 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_CS_MSR);
4713 }
4714
4715 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_EIP_MSR)
4716 {
4717 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, pMixedCtx->SysEnter.eip);
4718 AssertRCReturn(rc, rc);
4719 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_EIP_MSR);
4720 }
4721
4722 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_SYSENTER_ESP_MSR)
4723 {
4724 int rc = VMXWriteVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, pMixedCtx->SysEnter.esp);
4725 AssertRCReturn(rc, rc);
4726 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_SYSENTER_ESP_MSR);
4727 }
4728 }
4729
4730 if (ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged) & HM_CHANGED_GUEST_EFER_MSR)
4731 {
4732 if (hmR0VmxShouldSwapEferMsr(pVCpu, pMixedCtx))
4733 {
4734 /*
4735 * If the CPU supports VMCS controls for swapping EFER, use it. Otherwise, we have no option
4736 * but to use the auto-load store MSR area in the VMCS for swapping EFER. See @bugref{7368}.
4737 */
4738 if (pVM->hm.s.vmx.fSupportsVmcsEfer)
4739 {
4740 int rc = VMXWriteVmcs64(VMX_VMCS64_GUEST_EFER_FULL, pMixedCtx->msrEFER);
4741 AssertRCReturn(rc,rc);
4742 Log4Func(("EFER=%#RX64\n", pMixedCtx->msrEFER));
4743 }
4744 else
4745 {
4746 int rc = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K6_EFER, pMixedCtx->msrEFER, false /* fUpdateHostMsr */,
4747 NULL /* pfAddedAndUpdated */);
4748 AssertRCReturn(rc, rc);
4749
4750 /* We need to intercept reads too, see @bugref{7386#c16}. */
4751 if (pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
4752 hmR0VmxSetMsrPermission(pVCpu, MSR_K6_EFER, VMXMSREXIT_INTERCEPT_READ, VMXMSREXIT_INTERCEPT_WRITE);
4753 Log4Func(("MSR[--]: u32Msr=%#RX32 u64Value=%#RX64 cMsrs=%u\n", MSR_K6_EFER, pMixedCtx->msrEFER,
4754 pVCpu->hm.s.vmx.cMsrs));
4755 }
4756 }
4757 else if (!pVM->hm.s.vmx.fSupportsVmcsEfer)
4758 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K6_EFER);
4759 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~HM_CHANGED_GUEST_EFER_MSR);
4760 }
4761
4762 return VINF_SUCCESS;
4763}
4764
4765
4766#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
4767/**
4768 * Check if guest state allows safe use of 32-bit switcher again.
4769 *
4770 * Segment bases and protected mode structures must be 32-bit addressable
4771 * because the 32-bit switcher will ignore high dword when writing these VMCS
4772 * fields. See @bugref{8432} for details.
4773 *
4774 * @returns true if safe, false if must continue to use the 64-bit switcher.
4775 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4776 * out-of-sync. Make sure to update the required fields
4777 * before using them.
4778 *
4779 * @remarks No-long-jump zone!!!
4780 */
4781static bool hmR0VmxIs32BitSwitcherSafe(PCCPUMCTX pMixedCtx)
4782{
4783 if (pMixedCtx->gdtr.pGdt & UINT64_C(0xffffffff00000000)) return false;
4784 if (pMixedCtx->idtr.pIdt & UINT64_C(0xffffffff00000000)) return false;
4785 if (pMixedCtx->ldtr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4786 if (pMixedCtx->tr.u64Base & UINT64_C(0xffffffff00000000)) return false;
4787 if (pMixedCtx->es.u64Base & UINT64_C(0xffffffff00000000)) return false;
4788 if (pMixedCtx->cs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4789 if (pMixedCtx->ss.u64Base & UINT64_C(0xffffffff00000000)) return false;
4790 if (pMixedCtx->ds.u64Base & UINT64_C(0xffffffff00000000)) return false;
4791 if (pMixedCtx->fs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4792 if (pMixedCtx->gs.u64Base & UINT64_C(0xffffffff00000000)) return false;
4793
4794 /* All good, bases are 32-bit. */
4795 return true;
4796}
4797#endif
4798
4799
4800/**
4801 * Selects up the appropriate function to run guest code.
4802 *
4803 * @returns VBox status code.
4804 * @param pVCpu The cross context virtual CPU structure.
4805 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
4806 * out-of-sync. Make sure to update the required fields
4807 * before using them.
4808 *
4809 * @remarks No-long-jump zone!!!
4810 */
4811static int hmR0VmxSelectVMRunHandler(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
4812{
4813 if (CPUMIsGuestInLongModeEx(pMixedCtx))
4814 {
4815#ifndef VBOX_ENABLE_64_BITS_GUESTS
4816 return VERR_PGM_UNSUPPORTED_SHADOW_PAGING_MODE;
4817#endif
4818 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests); /* Guaranteed by hmR3InitFinalizeR0(). */
4819#if HC_ARCH_BITS == 32
4820 /* 32-bit host. We need to switch to 64-bit before running the 64-bit guest. */
4821 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
4822 {
4823#ifdef VBOX_STRICT
4824 if (pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4825 {
4826 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4827 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4828 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4829 | HM_CHANGED_VMX_ENTRY_CTLS
4830 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4831 }
4832#endif
4833 pVCpu->hm.s.vmx.pfnStartVM = VMXR0SwitcherStartVM64;
4834
4835 /* Mark that we've switched to 64-bit handler, we can't safely switch back to 32-bit for
4836 the rest of the VM run (until VM reset). See @bugref{8432#c7}. */
4837 pVCpu->hm.s.vmx.fSwitchedTo64on32 = true;
4838 Log4Func(("Selected 64-bit switcher\n"));
4839 }
4840#else
4841 /* 64-bit host. */
4842 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM64;
4843#endif
4844 }
4845 else
4846 {
4847 /* Guest is not in long mode, use the 32-bit handler. */
4848#if HC_ARCH_BITS == 32
4849 if ( pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32
4850 && !pVCpu->hm.s.vmx.fSwitchedTo64on32 /* If set, guest mode change does not imply switcher change. */
4851 && pVCpu->hm.s.vmx.pfnStartVM != NULL) /* Very first entry would have saved host-state already, ignore it. */
4852 {
4853# ifdef VBOX_STRICT
4854 /* Currently, all mode changes sends us back to ring-3, so these should be set. See @bugref{6944}. */
4855 uint64_t const fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
4856 AssertMsg(fCtxChanged & ( HM_CHANGED_VMX_EXIT_CTLS
4857 | HM_CHANGED_VMX_ENTRY_CTLS
4858 | HM_CHANGED_GUEST_EFER_MSR), ("fCtxChanged=%#RX64\n", fCtxChanged));
4859# endif
4860 }
4861# ifdef VBOX_ENABLE_64_BITS_GUESTS
4862 /*
4863 * Keep using the 64-bit switcher even though we're in 32-bit because of bad Intel
4864 * design, see @bugref{8432#c7}. If real-on-v86 mode is active, clear the 64-bit
4865 * switcher flag because now we know the guest is in a sane state where it's safe
4866 * to use the 32-bit switcher. Otherwise check the guest state if it's safe to use
4867 * the much faster 32-bit switcher again.
4868 */
4869 if (!pVCpu->hm.s.vmx.fSwitchedTo64on32)
4870 {
4871 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0StartVM32)
4872 Log4Func(("Selected 32-bit switcher\n"));
4873 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4874 }
4875 else
4876 {
4877 Assert(pVCpu->hm.s.vmx.pfnStartVM == VMXR0SwitcherStartVM64);
4878 if ( pVCpu->hm.s.vmx.RealMode.fRealOnV86Active
4879 || hmR0VmxIs32BitSwitcherSafe(pMixedCtx))
4880 {
4881 pVCpu->hm.s.vmx.fSwitchedTo64on32 = false;
4882 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4883 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
4884 | HM_CHANGED_VMX_ENTRY_CTLS
4885 | HM_CHANGED_VMX_EXIT_CTLS
4886 | HM_CHANGED_HOST_CONTEXT);
4887 Log4Func(("Selected 32-bit switcher (safe)\n"));
4888 }
4889 }
4890# else
4891 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4892# endif
4893#else
4894 pVCpu->hm.s.vmx.pfnStartVM = VMXR0StartVM32;
4895#endif
4896 }
4897 Assert(pVCpu->hm.s.vmx.pfnStartVM);
4898 return VINF_SUCCESS;
4899}
4900
4901
4902/**
4903 * Wrapper for running the guest code in VT-x.
4904 *
4905 * @returns VBox status code, no informational status codes.
4906 * @param pVCpu The cross context virtual CPU structure.
4907 * @param pCtx Pointer to the guest-CPU context.
4908 *
4909 * @remarks No-long-jump zone!!!
4910 */
4911DECLINLINE(int) hmR0VmxRunGuest(PVMCPU pVCpu, PCPUMCTX pCtx)
4912{
4913 /* Mark that HM is the keeper of all guest-CPU registers now that we're going to execute guest code. */
4914 pCtx->fExtrn |= HMVMX_CPUMCTX_EXTRN_ALL | CPUMCTX_EXTRN_KEEPER_HM;
4915
4916 /*
4917 * 64-bit Windows uses XMM registers in the kernel as the Microsoft compiler expresses
4918 * floating-point operations using SSE instructions. Some XMM registers (XMM6-XMM15) are
4919 * callee-saved and thus the need for this XMM wrapper.
4920 *
4921 * See MSDN "Configuring Programs for 64-bit/x64 Software Conventions / Register Usage".
4922 */
4923 bool const fResumeVM = RT_BOOL(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED);
4924 /** @todo Add stats for resume vs launch. */
4925 PVM pVM = pVCpu->CTX_SUFF(pVM);
4926#ifdef VBOX_WITH_KERNEL_USING_XMM
4927 int rc = hmR0VMXStartVMWrapXMM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu, pVCpu->hm.s.vmx.pfnStartVM);
4928#else
4929 int rc = pVCpu->hm.s.vmx.pfnStartVM(fResumeVM, pCtx, &pVCpu->hm.s.vmx.VMCSCache, pVM, pVCpu);
4930#endif
4931 AssertMsg(rc <= VINF_SUCCESS, ("%Rrc\n", rc));
4932 return rc;
4933}
4934
4935
4936/**
4937 * Reports world-switch error and dumps some useful debug info.
4938 *
4939 * @param pVCpu The cross context virtual CPU structure.
4940 * @param rcVMRun The return code from VMLAUNCH/VMRESUME.
4941 * @param pCtx Pointer to the guest-CPU context.
4942 * @param pVmxTransient Pointer to the VMX transient structure (only
4943 * exitReason updated).
4944 */
4945static void hmR0VmxReportWorldSwitchError(PVMCPU pVCpu, int rcVMRun, PCPUMCTX pCtx, PVMXTRANSIENT pVmxTransient)
4946{
4947 Assert(pVCpu);
4948 Assert(pCtx);
4949 Assert(pVmxTransient);
4950 HMVMX_ASSERT_PREEMPT_SAFE();
4951
4952 Log4Func(("VM-entry failure: %Rrc\n", rcVMRun));
4953 switch (rcVMRun)
4954 {
4955 case VERR_VMX_INVALID_VMXON_PTR:
4956 AssertFailed();
4957 break;
4958 case VINF_SUCCESS: /* VMLAUNCH/VMRESUME succeeded but VM-entry failed... yeah, true story. */
4959 case VERR_VMX_UNABLE_TO_START_VM: /* VMLAUNCH/VMRESUME itself failed. */
4960 {
4961 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &pVCpu->hm.s.vmx.LastError.u32ExitReason);
4962 rc |= VMXReadVmcs32(VMX_VMCS32_RO_VM_INSTR_ERROR, &pVCpu->hm.s.vmx.LastError.u32InstrError);
4963 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
4964 AssertRC(rc);
4965
4966 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
4967 /* LastError.idCurrentCpu was already updated in hmR0VmxPreRunGuestCommitted().
4968 Cannot do it here as we may have been long preempted. */
4969
4970#ifdef VBOX_STRICT
4971 Log4(("uExitReason %#RX32 (VmxTransient %#RX16)\n", pVCpu->hm.s.vmx.LastError.u32ExitReason,
4972 pVmxTransient->uExitReason));
4973 Log4(("Exit Qualification %#RX64\n", pVmxTransient->uExitQualification));
4974 Log4(("InstrError %#RX32\n", pVCpu->hm.s.vmx.LastError.u32InstrError));
4975 if (pVCpu->hm.s.vmx.LastError.u32InstrError <= HMVMX_INSTR_ERROR_MAX)
4976 Log4(("InstrError Desc. \"%s\"\n", g_apszVmxInstrErrors[pVCpu->hm.s.vmx.LastError.u32InstrError]));
4977 else
4978 Log4(("InstrError Desc. Range exceeded %u\n", HMVMX_INSTR_ERROR_MAX));
4979 Log4(("Entered host CPU %u\n", pVCpu->hm.s.vmx.LastError.idEnteredCpu));
4980 Log4(("Current host CPU %u\n", pVCpu->hm.s.vmx.LastError.idCurrentCpu));
4981
4982 /* VMX control bits. */
4983 uint32_t u32Val;
4984 uint64_t u64Val;
4985 RTHCUINTREG uHCReg;
4986 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PIN_EXEC, &u32Val); AssertRC(rc);
4987 Log4(("VMX_VMCS32_CTRL_PIN_EXEC %#RX32\n", u32Val));
4988 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, &u32Val); AssertRC(rc);
4989 Log4(("VMX_VMCS32_CTRL_PROC_EXEC %#RX32\n", u32Val));
4990 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL)
4991 {
4992 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, &u32Val); AssertRC(rc);
4993 Log4(("VMX_VMCS32_CTRL_PROC_EXEC2 %#RX32\n", u32Val));
4994 }
4995 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val); AssertRC(rc);
4996 Log4(("VMX_VMCS32_CTRL_ENTRY %#RX32\n", u32Val));
4997 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT, &u32Val); AssertRC(rc);
4998 Log4(("VMX_VMCS32_CTRL_EXIT %#RX32\n", u32Val));
4999 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_CR3_TARGET_COUNT, &u32Val); AssertRC(rc);
5000 Log4(("VMX_VMCS32_CTRL_CR3_TARGET_COUNT %#RX32\n", u32Val));
5001 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32Val); AssertRC(rc);
5002 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", u32Val));
5003 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, &u32Val); AssertRC(rc);
5004 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", u32Val));
5005 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, &u32Val); AssertRC(rc);
5006 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %u\n", u32Val));
5007 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_TPR_THRESHOLD, &u32Val); AssertRC(rc);
5008 Log4(("VMX_VMCS32_CTRL_TPR_THRESHOLD %u\n", u32Val));
5009 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT, &u32Val); AssertRC(rc);
5010 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_STORE_COUNT %u (guest MSRs)\n", u32Val));
5011 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5012 Log4(("VMX_VMCS32_CTRL_EXIT_MSR_LOAD_COUNT %u (host MSRs)\n", u32Val));
5013 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT, &u32Val); AssertRC(rc);
5014 Log4(("VMX_VMCS32_CTRL_ENTRY_MSR_LOAD_COUNT %u (guest MSRs)\n", u32Val));
5015 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, &u32Val); AssertRC(rc);
5016 Log4(("VMX_VMCS32_CTRL_EXCEPTION_BITMAP %#RX32\n", u32Val));
5017 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK, &u32Val); AssertRC(rc);
5018 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MASK %#RX32\n", u32Val));
5019 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH, &u32Val); AssertRC(rc);
5020 Log4(("VMX_VMCS32_CTRL_PAGEFAULT_ERROR_MATCH %#RX32\n", u32Val));
5021 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
5022 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
5023 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
5024 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5025 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
5026 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
5027 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
5028 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
5029 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5030 {
5031 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
5032 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
5033 }
5034
5035 /* Guest bits. */
5036 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val); AssertRC(rc);
5037 Log4(("Old Guest Rip %#RX64 New %#RX64\n", pCtx->rip, u64Val));
5038 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val); AssertRC(rc);
5039 Log4(("Old Guest Rsp %#RX64 New %#RX64\n", pCtx->rsp, u64Val));
5040 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val); AssertRC(rc);
5041 Log4(("Old Guest Rflags %#RX32 New %#RX32\n", pCtx->eflags.u32, u32Val));
5042 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fVpid)
5043 {
5044 rc = VMXReadVmcs32(VMX_VMCS16_VPID, &u32Val); AssertRC(rc);
5045 Log4(("VMX_VMCS16_VPID %u\n", u32Val));
5046 }
5047
5048 /* Host bits. */
5049 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR0, &uHCReg); AssertRC(rc);
5050 Log4(("Host CR0 %#RHr\n", uHCReg));
5051 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR3, &uHCReg); AssertRC(rc);
5052 Log4(("Host CR3 %#RHr\n", uHCReg));
5053 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_CR4, &uHCReg); AssertRC(rc);
5054 Log4(("Host CR4 %#RHr\n", uHCReg));
5055
5056 RTGDTR HostGdtr;
5057 PCX86DESCHC pDesc;
5058 ASMGetGDTR(&HostGdtr);
5059 rc = VMXReadVmcs32(VMX_VMCS16_HOST_CS_SEL, &u32Val); AssertRC(rc);
5060 Log4(("Host CS %#08x\n", u32Val));
5061 if (u32Val < HostGdtr.cbGdt)
5062 {
5063 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5064 hmR0DumpDescriptor(pDesc, u32Val, "CS: ");
5065 }
5066
5067 rc = VMXReadVmcs32(VMX_VMCS16_HOST_DS_SEL, &u32Val); AssertRC(rc);
5068 Log4(("Host DS %#08x\n", u32Val));
5069 if (u32Val < HostGdtr.cbGdt)
5070 {
5071 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5072 hmR0DumpDescriptor(pDesc, u32Val, "DS: ");
5073 }
5074
5075 rc = VMXReadVmcs32(VMX_VMCS16_HOST_ES_SEL, &u32Val); AssertRC(rc);
5076 Log4(("Host ES %#08x\n", u32Val));
5077 if (u32Val < HostGdtr.cbGdt)
5078 {
5079 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5080 hmR0DumpDescriptor(pDesc, u32Val, "ES: ");
5081 }
5082
5083 rc = VMXReadVmcs32(VMX_VMCS16_HOST_FS_SEL, &u32Val); AssertRC(rc);
5084 Log4(("Host FS %#08x\n", u32Val));
5085 if (u32Val < HostGdtr.cbGdt)
5086 {
5087 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5088 hmR0DumpDescriptor(pDesc, u32Val, "FS: ");
5089 }
5090
5091 rc = VMXReadVmcs32(VMX_VMCS16_HOST_GS_SEL, &u32Val); AssertRC(rc);
5092 Log4(("Host GS %#08x\n", u32Val));
5093 if (u32Val < HostGdtr.cbGdt)
5094 {
5095 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5096 hmR0DumpDescriptor(pDesc, u32Val, "GS: ");
5097 }
5098
5099 rc = VMXReadVmcs32(VMX_VMCS16_HOST_SS_SEL, &u32Val); AssertRC(rc);
5100 Log4(("Host SS %#08x\n", u32Val));
5101 if (u32Val < HostGdtr.cbGdt)
5102 {
5103 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5104 hmR0DumpDescriptor(pDesc, u32Val, "SS: ");
5105 }
5106
5107 rc = VMXReadVmcs32(VMX_VMCS16_HOST_TR_SEL, &u32Val); AssertRC(rc);
5108 Log4(("Host TR %#08x\n", u32Val));
5109 if (u32Val < HostGdtr.cbGdt)
5110 {
5111 pDesc = (PCX86DESCHC)(HostGdtr.pGdt + (u32Val & X86_SEL_MASK));
5112 hmR0DumpDescriptor(pDesc, u32Val, "TR: ");
5113 }
5114
5115 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_TR_BASE, &uHCReg); AssertRC(rc);
5116 Log4(("Host TR Base %#RHv\n", uHCReg));
5117 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_GDTR_BASE, &uHCReg); AssertRC(rc);
5118 Log4(("Host GDTR Base %#RHv\n", uHCReg));
5119 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_IDTR_BASE, &uHCReg); AssertRC(rc);
5120 Log4(("Host IDTR Base %#RHv\n", uHCReg));
5121 rc = VMXReadVmcs32(VMX_VMCS32_HOST_SYSENTER_CS, &u32Val); AssertRC(rc);
5122 Log4(("Host SYSENTER CS %#08x\n", u32Val));
5123 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_EIP, &uHCReg); AssertRC(rc);
5124 Log4(("Host SYSENTER EIP %#RHv\n", uHCReg));
5125 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_SYSENTER_ESP, &uHCReg); AssertRC(rc);
5126 Log4(("Host SYSENTER ESP %#RHv\n", uHCReg));
5127 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RSP, &uHCReg); AssertRC(rc);
5128 Log4(("Host RSP %#RHv\n", uHCReg));
5129 rc = VMXReadVmcsHstN(VMX_VMCS_HOST_RIP, &uHCReg); AssertRC(rc);
5130 Log4(("Host RIP %#RHv\n", uHCReg));
5131# if HC_ARCH_BITS == 64
5132 Log4(("MSR_K6_EFER = %#RX64\n", ASMRdMsr(MSR_K6_EFER)));
5133 Log4(("MSR_K8_CSTAR = %#RX64\n", ASMRdMsr(MSR_K8_CSTAR)));
5134 Log4(("MSR_K8_LSTAR = %#RX64\n", ASMRdMsr(MSR_K8_LSTAR)));
5135 Log4(("MSR_K6_STAR = %#RX64\n", ASMRdMsr(MSR_K6_STAR)));
5136 Log4(("MSR_K8_SF_MASK = %#RX64\n", ASMRdMsr(MSR_K8_SF_MASK)));
5137 Log4(("MSR_K8_KERNEL_GS_BASE = %#RX64\n", ASMRdMsr(MSR_K8_KERNEL_GS_BASE)));
5138# endif
5139#endif /* VBOX_STRICT */
5140 break;
5141 }
5142
5143 default:
5144 /* Impossible */
5145 AssertMsgFailed(("hmR0VmxReportWorldSwitchError %Rrc (%#x)\n", rcVMRun, rcVMRun));
5146 break;
5147 }
5148 NOREF(pCtx);
5149}
5150
5151
5152#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
5153#ifndef VMX_USE_CACHED_VMCS_ACCESSES
5154# error "VMX_USE_CACHED_VMCS_ACCESSES not defined when it should be!"
5155#endif
5156#ifdef VBOX_STRICT
5157static bool hmR0VmxIsValidWriteField(uint32_t idxField)
5158{
5159 switch (idxField)
5160 {
5161 case VMX_VMCS_GUEST_RIP:
5162 case VMX_VMCS_GUEST_RSP:
5163 case VMX_VMCS_GUEST_SYSENTER_EIP:
5164 case VMX_VMCS_GUEST_SYSENTER_ESP:
5165 case VMX_VMCS_GUEST_GDTR_BASE:
5166 case VMX_VMCS_GUEST_IDTR_BASE:
5167 case VMX_VMCS_GUEST_CS_BASE:
5168 case VMX_VMCS_GUEST_DS_BASE:
5169 case VMX_VMCS_GUEST_ES_BASE:
5170 case VMX_VMCS_GUEST_FS_BASE:
5171 case VMX_VMCS_GUEST_GS_BASE:
5172 case VMX_VMCS_GUEST_SS_BASE:
5173 case VMX_VMCS_GUEST_LDTR_BASE:
5174 case VMX_VMCS_GUEST_TR_BASE:
5175 case VMX_VMCS_GUEST_CR3:
5176 return true;
5177 }
5178 return false;
5179}
5180
5181static bool hmR0VmxIsValidReadField(uint32_t idxField)
5182{
5183 switch (idxField)
5184 {
5185 /* Read-only fields. */
5186 case VMX_VMCS_RO_EXIT_QUALIFICATION:
5187 return true;
5188 }
5189 /* Remaining readable fields should also be writable. */
5190 return hmR0VmxIsValidWriteField(idxField);
5191}
5192#endif /* VBOX_STRICT */
5193
5194
5195/**
5196 * Executes the specified handler in 64-bit mode.
5197 *
5198 * @returns VBox status code (no informational status codes).
5199 * @param pVCpu The cross context virtual CPU structure.
5200 * @param enmOp The operation to perform.
5201 * @param cParams Number of parameters.
5202 * @param paParam Array of 32-bit parameters.
5203 */
5204VMMR0DECL(int) VMXR0Execute64BitsHandler(PVMCPU pVCpu, HM64ON32OP enmOp, uint32_t cParams, uint32_t *paParam)
5205{
5206 PVM pVM = pVCpu->CTX_SUFF(pVM);
5207 AssertReturn(pVM->hm.s.pfnHost32ToGuest64R0, VERR_HM_NO_32_TO_64_SWITCHER);
5208 Assert(enmOp > HM64ON32OP_INVALID && enmOp < HM64ON32OP_END);
5209 Assert(pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Write.aField));
5210 Assert(pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries <= RT_ELEMENTS(pVCpu->hm.s.vmx.VMCSCache.Read.aField));
5211
5212#ifdef VBOX_STRICT
5213 for (uint32_t i = 0; i < pVCpu->hm.s.vmx.VMCSCache.Write.cValidEntries; i++)
5214 Assert(hmR0VmxIsValidWriteField(pVCpu->hm.s.vmx.VMCSCache.Write.aField[i]));
5215
5216 for (uint32_t i = 0; i <pVCpu->hm.s.vmx.VMCSCache.Read.cValidEntries; i++)
5217 Assert(hmR0VmxIsValidReadField(pVCpu->hm.s.vmx.VMCSCache.Read.aField[i]));
5218#endif
5219
5220 /* Disable interrupts. */
5221 RTCCUINTREG fOldEFlags = ASMIntDisableFlags();
5222
5223#ifdef VBOX_WITH_VMMR0_DISABLE_LAPIC_NMI
5224 RTCPUID idHostCpu = RTMpCpuId();
5225 CPUMR0SetLApic(pVCpu, idHostCpu);
5226#endif
5227
5228 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5229 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5230
5231 /* Clear VMCS. Marking it inactive, clearing implementation-specific data and writing VMCS data back to memory. */
5232 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5233 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
5234
5235 /* Leave VMX Root Mode. */
5236 VMXDisable();
5237
5238 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5239
5240 CPUMSetHyperESP(pVCpu, VMMGetStackRC(pVCpu));
5241 CPUMSetHyperEIP(pVCpu, enmOp);
5242 for (int i = (int)cParams - 1; i >= 0; i--)
5243 CPUMPushHyper(pVCpu, paParam[i]);
5244
5245 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatWorldSwitch3264, z);
5246
5247 /* Call the switcher. */
5248 int rc = pVM->hm.s.pfnHost32ToGuest64R0(pVM, RT_OFFSETOF(VM, aCpus[pVCpu->idCpu].cpum) - RT_OFFSETOF(VM, cpum));
5249 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatWorldSwitch3264, z);
5250
5251 /** @todo replace with hmR0VmxEnterRootMode() and hmR0VmxLeaveRootMode(). */
5252 /* Make sure the VMX instructions don't cause #UD faults. */
5253 SUPR0ChangeCR4(X86_CR4_VMXE, RTCCUINTREG_MAX);
5254
5255 /* Re-enter VMX Root Mode */
5256 int rc2 = VMXEnable(HCPhysCpuPage);
5257 if (RT_FAILURE(rc2))
5258 {
5259 SUPR0ChangeCR4(0, ~X86_CR4_VMXE);
5260 ASMSetFlags(fOldEFlags);
5261 pVM->hm.s.vmx.HCPhysVmxEnableError = HCPhysCpuPage;
5262 return rc2;
5263 }
5264
5265 rc2 = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
5266 AssertRC(rc2);
5267 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
5268 Assert(!(ASMGetFlags() & X86_EFL_IF));
5269 ASMSetFlags(fOldEFlags);
5270 return rc;
5271}
5272
5273
5274/**
5275 * Prepares for and executes VMLAUNCH (64-bit guests) for 32-bit hosts
5276 * supporting 64-bit guests.
5277 *
5278 * @returns VBox status code.
5279 * @param fResume Whether to VMLAUNCH or VMRESUME.
5280 * @param pCtx Pointer to the guest-CPU context.
5281 * @param pCache Pointer to the VMCS cache.
5282 * @param pVM The cross context VM structure.
5283 * @param pVCpu The cross context virtual CPU structure.
5284 */
5285DECLASM(int) VMXR0SwitcherStartVM64(RTHCUINT fResume, PCPUMCTX pCtx, PVMCSCACHE pCache, PVM pVM, PVMCPU pVCpu)
5286{
5287 NOREF(fResume);
5288
5289 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
5290 RTHCPHYS HCPhysCpuPage = pCpu->HCPhysMemObj;
5291
5292#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5293 pCache->uPos = 1;
5294 pCache->interPD = PGMGetInterPaeCR3(pVM);
5295 pCache->pSwitcher = (uint64_t)pVM->hm.s.pfnHost32ToGuest64R0;
5296#endif
5297
5298#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5299 pCache->TestIn.HCPhysCpuPage = 0;
5300 pCache->TestIn.HCPhysVmcs = 0;
5301 pCache->TestIn.pCache = 0;
5302 pCache->TestOut.HCPhysVmcs = 0;
5303 pCache->TestOut.pCache = 0;
5304 pCache->TestOut.pCtx = 0;
5305 pCache->TestOut.eflags = 0;
5306#else
5307 NOREF(pCache);
5308#endif
5309
5310 uint32_t aParam[10];
5311 aParam[0] = RT_LO_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Lo. */
5312 aParam[1] = RT_HI_U32(HCPhysCpuPage); /* Param 1: VMXON physical address - Hi. */
5313 aParam[2] = RT_LO_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Lo. */
5314 aParam[3] = RT_HI_U32(pVCpu->hm.s.vmx.HCPhysVmcs); /* Param 2: VMCS physical address - Hi. */
5315 aParam[4] = VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache);
5316 aParam[5] = 0;
5317 aParam[6] = VM_RC_ADDR(pVM, pVM);
5318 aParam[7] = 0;
5319 aParam[8] = VM_RC_ADDR(pVM, pVCpu);
5320 aParam[9] = 0;
5321
5322#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5323 pCtx->dr[4] = pVM->hm.s.vmx.pScratchPhys + 16 + 8;
5324 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 1;
5325#endif
5326 int rc = VMXR0Execute64BitsHandler(pVCpu, HM64ON32OP_VMXRCStartVM64, RT_ELEMENTS(aParam), &aParam[0]);
5327
5328#ifdef VBOX_WITH_CRASHDUMP_MAGIC
5329 Assert(*(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) == 5);
5330 Assert(pCtx->dr[4] == 10);
5331 *(uint32_t *)(pVM->hm.s.vmx.pScratch + 16 + 8) = 0xff;
5332#endif
5333
5334#if defined(DEBUG) && defined(VMX_USE_CACHED_VMCS_ACCESSES)
5335 AssertMsg(pCache->TestIn.HCPhysCpuPage == HCPhysCpuPage, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysCpuPage, HCPhysCpuPage));
5336 AssertMsg(pCache->TestIn.HCPhysVmcs == pVCpu->hm.s.vmx.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5337 pVCpu->hm.s.vmx.HCPhysVmcs));
5338 AssertMsg(pCache->TestIn.HCPhysVmcs == pCache->TestOut.HCPhysVmcs, ("%RHp vs %RHp\n", pCache->TestIn.HCPhysVmcs,
5339 pCache->TestOut.HCPhysVmcs));
5340 AssertMsg(pCache->TestIn.pCache == pCache->TestOut.pCache, ("%RGv vs %RGv\n", pCache->TestIn.pCache,
5341 pCache->TestOut.pCache));
5342 AssertMsg(pCache->TestIn.pCache == VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache),
5343 ("%RGv vs %RGv\n", pCache->TestIn.pCache, VM_RC_ADDR(pVM, &pVM->aCpus[pVCpu->idCpu].hm.s.vmx.VMCSCache)));
5344 AssertMsg(pCache->TestIn.pCtx == pCache->TestOut.pCtx, ("%RGv vs %RGv\n", pCache->TestIn.pCtx,
5345 pCache->TestOut.pCtx));
5346 Assert(!(pCache->TestOut.eflags & X86_EFL_IF));
5347#endif
5348 NOREF(pCtx);
5349 return rc;
5350}
5351
5352
5353/**
5354 * Initialize the VMCS-Read cache.
5355 *
5356 * The VMCS cache is used for 32-bit hosts running 64-bit guests (except 32-bit
5357 * Darwin which runs with 64-bit paging in 32-bit mode) for 64-bit fields that
5358 * cannot be accessed in 32-bit mode. Some 64-bit fields -can- be accessed
5359 * (those that have a 32-bit FULL & HIGH part).
5360 *
5361 * @returns VBox status code.
5362 * @param pVCpu The cross context virtual CPU structure.
5363 */
5364static int hmR0VmxInitVmcsReadCache(PVMCPU pVCpu)
5365{
5366#define VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, idxField) \
5367 do { \
5368 Assert(pCache->Read.aField[idxField##_CACHE_IDX] == 0); \
5369 pCache->Read.aField[idxField##_CACHE_IDX] = idxField; \
5370 pCache->Read.aFieldVal[idxField##_CACHE_IDX] = 0; \
5371 ++cReadFields; \
5372 } while (0)
5373
5374 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5375 uint32_t cReadFields = 0;
5376
5377 /*
5378 * Don't remove the #if 0'd fields in this code. They're listed here for consistency
5379 * and serve to indicate exceptions to the rules.
5380 */
5381
5382 /* Guest-natural selector base fields. */
5383#if 0
5384 /* These are 32-bit in practice. See Intel spec. 2.5 "Control Registers". */
5385 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR0);
5386 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR4);
5387#endif
5388 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_ES_BASE);
5389 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CS_BASE);
5390 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SS_BASE);
5391 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_DS_BASE);
5392 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_FS_BASE);
5393 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GS_BASE);
5394 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_LDTR_BASE);
5395 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_TR_BASE);
5396 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_GDTR_BASE);
5397 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_IDTR_BASE);
5398 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RSP);
5399 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_RIP);
5400#if 0
5401 /* Unused natural width guest-state fields. */
5402 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS);
5403 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3); /* Handled in Nested Paging case */
5404#endif
5405 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_ESP);
5406 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_SYSENTER_EIP);
5407
5408 /* 64-bit guest-state fields; unused as we use two 32-bit VMREADs for
5409 these 64-bit fields (using "FULL" and "HIGH" fields). */
5410#if 0
5411 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL);
5412 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_DEBUGCTL_FULL);
5413 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PAT_FULL);
5414 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_EFER_FULL);
5415 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL);
5416 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE0_FULL);
5417 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE1_FULL);
5418 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE2_FULL);
5419 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS64_GUEST_PDPTE3_FULL);
5420#endif
5421
5422 /* Natural width guest-state fields. */
5423 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_QUALIFICATION);
5424#if 0
5425 /* Currently unused field. */
5426 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_RO_EXIT_GUEST_LINEAR_ADDR);
5427#endif
5428
5429 if (pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging)
5430 {
5431 VMXLOCAL_INIT_READ_CACHE_FIELD(pCache, VMX_VMCS_GUEST_CR3);
5432 AssertMsg(cReadFields == VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields,
5433 VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX));
5434 pCache->Read.cValidEntries = VMX_VMCS_MAX_NESTED_PAGING_CACHE_IDX;
5435 }
5436 else
5437 {
5438 AssertMsg(cReadFields == VMX_VMCS_MAX_CACHE_IDX, ("cReadFields=%u expected %u\n", cReadFields, VMX_VMCS_MAX_CACHE_IDX));
5439 pCache->Read.cValidEntries = VMX_VMCS_MAX_CACHE_IDX;
5440 }
5441
5442#undef VMXLOCAL_INIT_READ_CACHE_FIELD
5443 return VINF_SUCCESS;
5444}
5445
5446
5447/**
5448 * Writes a field into the VMCS. This can either directly invoke a VMWRITE or
5449 * queue up the VMWRITE by using the VMCS write cache (on 32-bit hosts, except
5450 * darwin, running 64-bit guests).
5451 *
5452 * @returns VBox status code.
5453 * @param pVCpu The cross context virtual CPU structure.
5454 * @param idxField The VMCS field encoding.
5455 * @param u64Val 16, 32 or 64-bit value.
5456 */
5457VMMR0DECL(int) VMXWriteVmcs64Ex(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5458{
5459 int rc;
5460 switch (idxField)
5461 {
5462 /*
5463 * These fields consists of a "FULL" and a "HIGH" part which can be written to individually.
5464 */
5465 /* 64-bit Control fields. */
5466 case VMX_VMCS64_CTRL_IO_BITMAP_A_FULL:
5467 case VMX_VMCS64_CTRL_IO_BITMAP_B_FULL:
5468 case VMX_VMCS64_CTRL_MSR_BITMAP_FULL:
5469 case VMX_VMCS64_CTRL_EXIT_MSR_STORE_FULL:
5470 case VMX_VMCS64_CTRL_EXIT_MSR_LOAD_FULL:
5471 case VMX_VMCS64_CTRL_ENTRY_MSR_LOAD_FULL:
5472 case VMX_VMCS64_CTRL_EXEC_VMCS_PTR_FULL:
5473 case VMX_VMCS64_CTRL_TSC_OFFSET_FULL:
5474 case VMX_VMCS64_CTRL_VAPIC_PAGEADDR_FULL:
5475 case VMX_VMCS64_CTRL_APIC_ACCESSADDR_FULL:
5476 case VMX_VMCS64_CTRL_VMFUNC_CTRLS_FULL:
5477 case VMX_VMCS64_CTRL_EPTP_FULL:
5478 case VMX_VMCS64_CTRL_EPTP_LIST_FULL:
5479 /* 64-bit Guest-state fields. */
5480 case VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL:
5481 case VMX_VMCS64_GUEST_DEBUGCTL_FULL:
5482 case VMX_VMCS64_GUEST_PAT_FULL:
5483 case VMX_VMCS64_GUEST_EFER_FULL:
5484 case VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL:
5485 case VMX_VMCS64_GUEST_PDPTE0_FULL:
5486 case VMX_VMCS64_GUEST_PDPTE1_FULL:
5487 case VMX_VMCS64_GUEST_PDPTE2_FULL:
5488 case VMX_VMCS64_GUEST_PDPTE3_FULL:
5489 /* 64-bit Host-state fields. */
5490 case VMX_VMCS64_HOST_PAT_FULL:
5491 case VMX_VMCS64_HOST_EFER_FULL:
5492 case VMX_VMCS64_HOST_PERF_GLOBAL_CTRL_FULL:
5493 {
5494 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5495 rc |= VMXWriteVmcs32(idxField + 1, RT_HI_U32(u64Val));
5496 break;
5497 }
5498
5499 /*
5500 * These fields do not have high and low parts. Queue up the VMWRITE by using the VMCS write-cache (for 64-bit
5501 * values). When we switch the host to 64-bit mode for running 64-bit guests, these VMWRITEs get executed then.
5502 */
5503 /* Natural-width Guest-state fields. */
5504 case VMX_VMCS_GUEST_CR3:
5505 case VMX_VMCS_GUEST_ES_BASE:
5506 case VMX_VMCS_GUEST_CS_BASE:
5507 case VMX_VMCS_GUEST_SS_BASE:
5508 case VMX_VMCS_GUEST_DS_BASE:
5509 case VMX_VMCS_GUEST_FS_BASE:
5510 case VMX_VMCS_GUEST_GS_BASE:
5511 case VMX_VMCS_GUEST_LDTR_BASE:
5512 case VMX_VMCS_GUEST_TR_BASE:
5513 case VMX_VMCS_GUEST_GDTR_BASE:
5514 case VMX_VMCS_GUEST_IDTR_BASE:
5515 case VMX_VMCS_GUEST_RSP:
5516 case VMX_VMCS_GUEST_RIP:
5517 case VMX_VMCS_GUEST_SYSENTER_ESP:
5518 case VMX_VMCS_GUEST_SYSENTER_EIP:
5519 {
5520 if (!(RT_HI_U32(u64Val)))
5521 {
5522 /* If this field is 64-bit, VT-x will zero out the top bits. */
5523 rc = VMXWriteVmcs32(idxField, RT_LO_U32(u64Val));
5524 }
5525 else
5526 {
5527 /* Assert that only the 32->64 switcher case should ever come here. */
5528 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fAllow64BitGuests);
5529 rc = VMXWriteCachedVmcsEx(pVCpu, idxField, u64Val);
5530 }
5531 break;
5532 }
5533
5534 default:
5535 {
5536 AssertMsgFailed(("VMXWriteVmcs64Ex: Invalid field %#RX32 (pVCpu=%p u64Val=%#RX64)\n", idxField, pVCpu, u64Val));
5537 rc = VERR_INVALID_PARAMETER;
5538 break;
5539 }
5540 }
5541 AssertRCReturn(rc, rc);
5542 return rc;
5543}
5544
5545
5546/**
5547 * Queue up a VMWRITE by using the VMCS write cache.
5548 * This is only used on 32-bit hosts (except darwin) for 64-bit guests.
5549 *
5550 * @param pVCpu The cross context virtual CPU structure.
5551 * @param idxField The VMCS field encoding.
5552 * @param u64Val 16, 32 or 64-bit value.
5553 */
5554VMMR0DECL(int) VMXWriteCachedVmcsEx(PVMCPU pVCpu, uint32_t idxField, uint64_t u64Val)
5555{
5556 AssertPtr(pVCpu);
5557 PVMCSCACHE pCache = &pVCpu->hm.s.vmx.VMCSCache;
5558
5559 AssertMsgReturn(pCache->Write.cValidEntries < VMCSCACHE_MAX_ENTRY - 1,
5560 ("entries=%u\n", pCache->Write.cValidEntries), VERR_ACCESS_DENIED);
5561
5562 /* Make sure there are no duplicates. */
5563 for (uint32_t i = 0; i < pCache->Write.cValidEntries; i++)
5564 {
5565 if (pCache->Write.aField[i] == idxField)
5566 {
5567 pCache->Write.aFieldVal[i] = u64Val;
5568 return VINF_SUCCESS;
5569 }
5570 }
5571
5572 pCache->Write.aField[pCache->Write.cValidEntries] = idxField;
5573 pCache->Write.aFieldVal[pCache->Write.cValidEntries] = u64Val;
5574 pCache->Write.cValidEntries++;
5575 return VINF_SUCCESS;
5576}
5577#endif /* HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS) */
5578
5579
5580/**
5581 * Sets up the usage of TSC-offsetting and updates the VMCS.
5582 *
5583 * If offsetting is not possible, cause VM-exits on RDTSC(P)s. Also sets up the
5584 * VMX preemption timer.
5585 *
5586 * @returns VBox status code.
5587 * @param pVCpu The cross context virtual CPU structure.
5588 *
5589 * @remarks No-long-jump zone!!!
5590 */
5591static void hmR0VmxUpdateTscOffsettingAndPreemptTimer(PVMCPU pVCpu)
5592{
5593 bool fOffsettedTsc;
5594 bool fParavirtTsc;
5595 PVM pVM = pVCpu->CTX_SUFF(pVM);
5596 uint64_t uTscOffset;
5597 if (pVM->hm.s.vmx.fUsePreemptTimer)
5598 {
5599 uint64_t cTicksToDeadline = TMCpuTickGetDeadlineAndTscOffset(pVM, pVCpu, &uTscOffset, &fOffsettedTsc, &fParavirtTsc);
5600
5601 /* Make sure the returned values have sane upper and lower boundaries. */
5602 uint64_t u64CpuHz = SUPGetCpuHzFromGipBySetIndex(g_pSUPGlobalInfoPage, pVCpu->iHostCpuSet);
5603 cTicksToDeadline = RT_MIN(cTicksToDeadline, u64CpuHz / 64); /* 1/64th of a second */
5604 cTicksToDeadline = RT_MAX(cTicksToDeadline, u64CpuHz / 2048); /* 1/2048th of a second */
5605 cTicksToDeadline >>= pVM->hm.s.vmx.cPreemptTimerShift;
5606
5607 uint32_t cPreemptionTickCount = (uint32_t)RT_MIN(cTicksToDeadline, UINT32_MAX - 16);
5608 int rc = VMXWriteVmcs32(VMX_VMCS32_GUEST_PREEMPT_TIMER_VALUE, cPreemptionTickCount);
5609 AssertRC(rc);
5610 }
5611 else
5612 fOffsettedTsc = TMCpuTickCanUseRealTSC(pVM, pVCpu, &uTscOffset, &fParavirtTsc);
5613
5614 /** @todo later optimize this to be done elsewhere and not before every
5615 * VM-entry. */
5616 if (fParavirtTsc)
5617 {
5618 /* Currently neither Hyper-V nor KVM need to update their paravirt. TSC
5619 information before every VM-entry, hence disable it for performance sake. */
5620#if 0
5621 int rc = GIMR0UpdateParavirtTsc(pVM, 0 /* u64Offset */);
5622 AssertRC(rc);
5623#endif
5624 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscParavirt);
5625 }
5626
5627 if ( fOffsettedTsc
5628 && RT_LIKELY(!pVCpu->hm.s.fDebugWantRdTscExit))
5629 {
5630 if (pVCpu->hm.s.vmx.u64TscOffset != uTscOffset)
5631 {
5632 int rc = VMXWriteVmcs64(VMX_VMCS64_CTRL_TSC_OFFSET_FULL, pVCpu->hm.s.vmx.u64TscOffset);
5633 AssertRC(rc);
5634 pVCpu->hm.s.vmx.u64TscOffset = uTscOffset;
5635 }
5636
5637 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT)
5638 {
5639 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5640 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5641 AssertRC(rc);
5642 }
5643 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscOffset);
5644 }
5645 else
5646 {
5647 /* We can't use TSC-offsetting (non-fixed TSC, warp drive active etc.), VM-exit on RDTSC(P). */
5648 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
5649 {
5650 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT;
5651 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
5652 AssertRC(rc);
5653 }
5654 STAM_COUNTER_INC(&pVCpu->hm.s.StatTscIntercept);
5655 }
5656}
5657
5658
5659/**
5660 * Gets the IEM exception flags for the specified vector and IDT vectoring /
5661 * VM-exit interruption info type.
5662 *
5663 * @returns The IEM exception flags.
5664 * @param uVector The event vector.
5665 * @param uVmxVectorType The VMX event type.
5666 *
5667 * @remarks This function currently only constructs flags required for
5668 * IEMEvaluateRecursiveXcpt and not the complete flags (e.g, error-code
5669 * and CR2 aspects of an exception are not included).
5670 */
5671static uint32_t hmR0VmxGetIemXcptFlags(uint8_t uVector, uint32_t uVmxVectorType)
5672{
5673 uint32_t fIemXcptFlags;
5674 switch (uVmxVectorType)
5675 {
5676 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
5677 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
5678 fIemXcptFlags = IEM_XCPT_FLAGS_T_CPU_XCPT;
5679 break;
5680
5681 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
5682 fIemXcptFlags = IEM_XCPT_FLAGS_T_EXT_INT;
5683 break;
5684
5685 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
5686 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT | IEM_XCPT_FLAGS_ICEBP_INSTR;
5687 break;
5688
5689 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT:
5690 {
5691 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5692 if (uVector == X86_XCPT_BP)
5693 fIemXcptFlags |= IEM_XCPT_FLAGS_BP_INSTR;
5694 else if (uVector == X86_XCPT_OF)
5695 fIemXcptFlags |= IEM_XCPT_FLAGS_OF_INSTR;
5696 else
5697 {
5698 fIemXcptFlags = 0;
5699 AssertMsgFailed(("Unexpected vector for software int. uVector=%#x", uVector));
5700 }
5701 break;
5702 }
5703
5704 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
5705 fIemXcptFlags = IEM_XCPT_FLAGS_T_SOFT_INT;
5706 break;
5707
5708 default:
5709 fIemXcptFlags = 0;
5710 AssertMsgFailed(("Unexpected vector type! uVmxVectorType=%#x uVector=%#x", uVmxVectorType, uVector));
5711 break;
5712 }
5713 return fIemXcptFlags;
5714}
5715
5716
5717/**
5718 * Sets an event as a pending event to be injected into the guest.
5719 *
5720 * @param pVCpu The cross context virtual CPU structure.
5721 * @param u32IntInfo The VM-entry interruption-information field.
5722 * @param cbInstr The VM-entry instruction length in bytes (for software
5723 * interrupts, exceptions and privileged software
5724 * exceptions).
5725 * @param u32ErrCode The VM-entry exception error code.
5726 * @param GCPtrFaultAddress The fault-address (CR2) in case it's a
5727 * page-fault.
5728 *
5729 * @remarks Statistics counter assumes this is a guest event being injected or
5730 * re-injected into the guest, i.e. 'StatInjectPendingReflect' is
5731 * always incremented.
5732 */
5733DECLINLINE(void) hmR0VmxSetPendingEvent(PVMCPU pVCpu, uint32_t u32IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
5734 RTGCUINTPTR GCPtrFaultAddress)
5735{
5736 Assert(!pVCpu->hm.s.Event.fPending);
5737 pVCpu->hm.s.Event.fPending = true;
5738 pVCpu->hm.s.Event.u64IntInfo = u32IntInfo;
5739 pVCpu->hm.s.Event.u32ErrCode = u32ErrCode;
5740 pVCpu->hm.s.Event.cbInstr = cbInstr;
5741 pVCpu->hm.s.Event.GCPtrFaultAddress = GCPtrFaultAddress;
5742}
5743
5744
5745/**
5746 * Sets a double-fault (\#DF) exception as pending-for-injection into the VM.
5747 *
5748 * @param pVCpu The cross context virtual CPU structure.
5749 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5750 * out-of-sync. Make sure to update the required fields
5751 * before using them.
5752 */
5753DECLINLINE(void) hmR0VmxSetPendingXcptDF(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
5754{
5755 NOREF(pMixedCtx);
5756 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
5757 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
5758 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
5759 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
5760}
5761
5762
5763/**
5764 * Handle a condition that occurred while delivering an event through the guest
5765 * IDT.
5766 *
5767 * @returns Strict VBox status code (i.e. informational status codes too).
5768 * @retval VINF_SUCCESS if we should continue handling the VM-exit.
5769 * @retval VINF_HM_DOUBLE_FAULT if a \#DF condition was detected and we ought
5770 * to continue execution of the guest which will delivery the \#DF.
5771 * @retval VINF_EM_RESET if we detected a triple-fault condition.
5772 * @retval VERR_EM_GUEST_CPU_HANG if we detected a guest CPU hang.
5773 *
5774 * @param pVCpu The cross context virtual CPU structure.
5775 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
5776 * out-of-sync. Make sure to update the required fields
5777 * before using them.
5778 * @param pVmxTransient Pointer to the VMX transient structure.
5779 *
5780 * @remarks No-long-jump zone!!!
5781 */
5782static VBOXSTRICTRC hmR0VmxCheckExitDueToEventDelivery(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
5783{
5784 uint32_t const uExitVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
5785
5786 int rc2 = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
5787 rc2 |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
5788 AssertRCReturn(rc2, rc2);
5789
5790 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
5791 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
5792 {
5793 uint32_t const uIdtVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
5794 uint32_t const uIdtVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
5795
5796 /*
5797 * If the event was a software interrupt (generated with INT n) or a software exception
5798 * (generated by INT3/INTO) or a privileged software exception (generated by INT1), we
5799 * can handle the VM-exit and continue guest execution which will re-execute the
5800 * instruction rather than re-injecting the exception, as that can cause premature
5801 * trips to ring-3 before injection and involve TRPM which currently has no way of
5802 * storing that these exceptions were caused by these instructions (ICEBP's #DB poses
5803 * the problem).
5804 */
5805 IEMXCPTRAISE enmRaise;
5806 IEMXCPTRAISEINFO fRaiseInfo;
5807 if ( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
5808 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
5809 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
5810 {
5811 enmRaise = IEMXCPTRAISE_REEXEC_INSTR;
5812 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5813 }
5814 else if (VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo))
5815 {
5816 uint32_t const uExitVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uExitIntInfo);
5817 uint32_t const fIdtVectorFlags = hmR0VmxGetIemXcptFlags(uIdtVector, uIdtVectorType);
5818 uint32_t const fExitVectorFlags = hmR0VmxGetIemXcptFlags(uExitVector, uExitVectorType);
5819 /** @todo Make AssertMsgReturn as just AssertMsg later. */
5820 AssertMsgReturn(uExitVectorType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT,
5821 ("hmR0VmxCheckExitDueToEventDelivery: Unexpected VM-exit interruption info. %#x!\n",
5822 uExitVectorType), VERR_VMX_IPE_5);
5823
5824 enmRaise = IEMEvaluateRecursiveXcpt(pVCpu, fIdtVectorFlags, uIdtVector, fExitVectorFlags, uExitVector, &fRaiseInfo);
5825
5826 /* Determine a vectoring #PF condition, see comment in hmR0VmxExitXcptPF(). */
5827 if (fRaiseInfo & (IEMXCPTRAISEINFO_EXT_INT_PF | IEMXCPTRAISEINFO_NMI_PF))
5828 {
5829 pVmxTransient->fVectoringPF = true;
5830 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5831 }
5832 }
5833 else
5834 {
5835 /*
5836 * If an exception or hardware interrupt delivery caused an EPT violation/misconfig or APIC access
5837 * VM-exit, then the VM-exit interruption-information will not be valid and we end up here.
5838 * It is sufficient to reflect the original event to the guest after handling the VM-exit.
5839 */
5840 Assert( uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
5841 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5842 || uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_EXT_INT);
5843 enmRaise = IEMXCPTRAISE_PREV_EVENT;
5844 fRaiseInfo = IEMXCPTRAISEINFO_NONE;
5845 }
5846
5847 /*
5848 * On CPUs that support Virtual NMIs, if this VM-exit (be it an exception or EPT violation/misconfig
5849 * etc.) occurred while delivering the NMI, we need to clear the block-by-NMI field in the guest
5850 * interruptibility-state before re-delivering the NMI after handling the VM-exit. Otherwise the
5851 * subsequent VM-entry would fail.
5852 *
5853 * See Intel spec. 30.7.1.2 "Resuming Guest Software after Handling an Exception". See @bugref{7445}.
5854 */
5855 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS)
5856 && uIdtVectorType == VMX_IDT_VECTORING_INFO_TYPE_NMI
5857 && ( enmRaise == IEMXCPTRAISE_PREV_EVENT
5858 || (fRaiseInfo & IEMXCPTRAISEINFO_NMI_PF))
5859 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5860 {
5861 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
5862 }
5863
5864 switch (enmRaise)
5865 {
5866 case IEMXCPTRAISE_CURRENT_XCPT:
5867 {
5868 Log4Func(("IDT: Pending secondary Xcpt: uIdtVectoringInfo=%#RX64 uExitIntInfo=%#RX64\n",
5869 pVmxTransient->uIdtVectoringInfo, pVmxTransient->uExitIntInfo));
5870 Assert(rcStrict == VINF_SUCCESS);
5871 break;
5872 }
5873
5874 case IEMXCPTRAISE_PREV_EVENT:
5875 {
5876 uint32_t u32ErrCode;
5877 if (VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo))
5878 {
5879 rc2 = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
5880 AssertRCReturn(rc2, rc2);
5881 u32ErrCode = pVmxTransient->uIdtVectoringErrorCode;
5882 }
5883 else
5884 u32ErrCode = 0;
5885
5886 /* If uExitVector is #PF, CR2 value will be updated from the VMCS if it's a guest #PF, see hmR0VmxExitXcptPF(). */
5887 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5888 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
5889 0 /* cbInstr */, u32ErrCode, pMixedCtx->cr2);
5890
5891 Log4Func(("IDT: Pending vectoring event %#RX64 Err=%#RX32\n", pVCpu->hm.s.Event.u64IntInfo,
5892 pVCpu->hm.s.Event.u32ErrCode));
5893 Assert(rcStrict == VINF_SUCCESS);
5894 break;
5895 }
5896
5897 case IEMXCPTRAISE_REEXEC_INSTR:
5898 Assert(rcStrict == VINF_SUCCESS);
5899 break;
5900
5901 case IEMXCPTRAISE_DOUBLE_FAULT:
5902 {
5903 /*
5904 * Determing a vectoring double #PF condition. Used later, when PGM evaluates the
5905 * second #PF as a guest #PF (and not a shadow #PF) and needs to be converted into a #DF.
5906 */
5907 if (fRaiseInfo & IEMXCPTRAISEINFO_PF_PF)
5908 {
5909 pVmxTransient->fVectoringDoublePF = true;
5910 Log4Func(("IDT: Vectoring double #PF %#RX64 cr2=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo,
5911 pMixedCtx->cr2));
5912 rcStrict = VINF_SUCCESS;
5913 }
5914 else
5915 {
5916 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingReflect);
5917 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
5918 Log4Func(("IDT: Pending vectoring #DF %#RX64 uIdtVector=%#x uExitVector=%#x\n", pVCpu->hm.s.Event.u64IntInfo,
5919 uIdtVector, uExitVector));
5920 rcStrict = VINF_HM_DOUBLE_FAULT;
5921 }
5922 break;
5923 }
5924
5925 case IEMXCPTRAISE_TRIPLE_FAULT:
5926 {
5927 Log4Func(("IDT: Pending vectoring triple-fault uIdt=%#x uExit=%#x\n", uIdtVector, uExitVector));
5928 rcStrict = VINF_EM_RESET;
5929 break;
5930 }
5931
5932 case IEMXCPTRAISE_CPU_HANG:
5933 {
5934 Log4Func(("IDT: Bad guest! Entering CPU hang. fRaiseInfo=%#x\n", fRaiseInfo));
5935 rcStrict = VERR_EM_GUEST_CPU_HANG;
5936 break;
5937 }
5938
5939 default:
5940 {
5941 AssertMsgFailed(("IDT: vcpu[%RU32] Unexpected/invalid value! enmRaise=%#x\n", pVCpu->idCpu, enmRaise));
5942 rcStrict = VERR_VMX_IPE_2;
5943 break;
5944 }
5945 }
5946 }
5947 else if ( VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo)
5948 && VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(pVmxTransient->uExitIntInfo)
5949 && uExitVector != X86_XCPT_DF
5950 && (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI))
5951 {
5952 /*
5953 * Execution of IRET caused this fault when NMI blocking was in effect (i.e we're in the guest NMI handler).
5954 * We need to set the block-by-NMI field so that NMIs remain blocked until the IRET execution is restarted.
5955 * See Intel spec. 30.7.1.2 "Resuming guest software after handling an exception".
5956 */
5957 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
5958 {
5959 Log4Func(("Setting VMCPU_FF_BLOCK_NMIS. fValid=%RTbool uExitReason=%u\n",
5960 VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo), pVmxTransient->uExitReason));
5961 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
5962 }
5963 }
5964
5965 Assert( rcStrict == VINF_SUCCESS || rcStrict == VINF_HM_DOUBLE_FAULT
5966 || rcStrict == VINF_EM_RESET || rcStrict == VERR_EM_GUEST_CPU_HANG);
5967 return rcStrict;
5968}
5969
5970
5971/**
5972 * Imports a guest segment register from the current VMCS into
5973 * the guest-CPU context.
5974 *
5975 * @returns VBox status code.
5976 * @param pVCpu The cross context virtual CPU structure.
5977 * @param idxSel Index of the selector in the VMCS.
5978 * @param idxLimit Index of the segment limit in the VMCS.
5979 * @param idxBase Index of the segment base in the VMCS.
5980 * @param idxAccess Index of the access rights of the segment in the VMCS.
5981 * @param pSelReg Pointer to the segment selector.
5982 *
5983 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
5984 * do not log!
5985 *
5986 * @remarks Never call this function directly!!! Use the
5987 * HMVMX_IMPORT_SREG() macro as that takes care
5988 * of whether to read from the VMCS cache or not.
5989 */
5990static int hmR0VmxImportGuestSegmentReg(PVMCPU pVCpu, uint32_t idxSel, uint32_t idxLimit, uint32_t idxBase, uint32_t idxAccess,
5991 PCPUMSELREG pSelReg)
5992{
5993 NOREF(pVCpu);
5994
5995 uint32_t u32Sel;
5996 uint32_t u32Limit;
5997 uint32_t u32Attr;
5998 uint64_t u64Base;
5999 int rc = VMXReadVmcs32(idxSel, &u32Sel);
6000 rc |= VMXReadVmcs32(idxLimit, &u32Limit);
6001 rc |= VMXReadVmcs32(idxAccess, &u32Attr);
6002 rc |= VMXReadVmcsGstNByIdxVal(idxBase, &u64Base);
6003 AssertRCReturn(rc, rc);
6004
6005 pSelReg->Sel = (uint16_t)u32Sel;
6006 pSelReg->ValidSel = (uint16_t)u32Sel;
6007 pSelReg->fFlags = CPUMSELREG_FLAGS_VALID;
6008 pSelReg->u32Limit = u32Limit;
6009 pSelReg->u64Base = u64Base;
6010 pSelReg->Attr.u = u32Attr;
6011
6012 /*
6013 * If VT-x marks the segment as unusable, most other bits remain undefined:
6014 * - For CS the L, D and G bits have meaning.
6015 * - For SS the DPL has meaning (it -is- the CPL for Intel and VBox).
6016 * - For the remaining data segments no bits are defined.
6017 *
6018 * The present bit and the unusable bit has been observed to be set at the
6019 * same time (the selector was supposed to be invalid as we started executing
6020 * a V8086 interrupt in ring-0).
6021 *
6022 * What should be important for the rest of the VBox code, is that the P bit is
6023 * cleared. Some of the other VBox code recognizes the unusable bit, but
6024 * AMD-V certainly don't, and REM doesn't really either. So, to be on the
6025 * safe side here, we'll strip off P and other bits we don't care about. If
6026 * any code breaks because Attr.u != 0 when Sel < 4, it should be fixed.
6027 *
6028 * See Intel spec. 27.3.2 "Saving Segment Registers and Descriptor-Table Registers".
6029 */
6030 if (pSelReg->Attr.u & X86DESCATTR_UNUSABLE)
6031 {
6032 Assert(idxSel != VMX_VMCS16_GUEST_TR_SEL); /* TR is the only selector that can never be unusable. */
6033
6034 /* Masking off: X86DESCATTR_P, X86DESCATTR_LIMIT_HIGH, and X86DESCATTR_AVL. The latter two are really irrelevant. */
6035 pSelReg->Attr.u &= X86DESCATTR_UNUSABLE | X86DESCATTR_L | X86DESCATTR_D | X86DESCATTR_G
6036 | X86DESCATTR_DPL | X86DESCATTR_TYPE | X86DESCATTR_DT;
6037#ifdef VBOX_STRICT
6038 VMMRZCallRing3Disable(pVCpu);
6039 Log4Func(("Unusable idxSel=%#x attr=%#x -> %#x\n", idxSel, u32Sel, pSelReg->Attr.u));
6040# ifdef DEBUG_bird
6041 AssertMsg((u32Attr & ~X86DESCATTR_P) == pSelReg->Attr.u,
6042 ("%#x: %#x != %#x (sel=%#x base=%#llx limit=%#x)\n",
6043 idxSel, u32Sel, pSelReg->Attr.u, pSelReg->Sel, pSelReg->u64Base, pSelReg->u32Limit));
6044# endif
6045 VMMRZCallRing3Enable(pVCpu);
6046#endif
6047 }
6048 return VINF_SUCCESS;
6049}
6050
6051
6052/**
6053 * Imports the guest RIP from the VMCS back into the guest-CPU context.
6054 *
6055 * @returns VBox status code.
6056 * @param pVCpu The cross context virtual CPU structure.
6057 *
6058 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6059 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6060 * instead!!!
6061 */
6062DECLINLINE(int) hmR0VmxImportGuestRip(PVMCPU pVCpu)
6063{
6064 uint64_t u64Val;
6065 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6066 if (pCtx->fExtrn & CPUMCTX_EXTRN_RIP)
6067 {
6068 int rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RIP, &u64Val);
6069 if (RT_SUCCESS(rc))
6070 {
6071 pCtx->rip = u64Val;
6072 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RIP;
6073 }
6074 return rc;
6075 }
6076 return VINF_SUCCESS;
6077}
6078
6079
6080/**
6081 * Imports the guest RFLAGS from the VMCS back into the guest-CPU context.
6082 *
6083 * @returns VBox status code.
6084 * @param pVCpu The cross context virtual CPU structure.
6085 *
6086 * @remarks Called with interrupts and/or preemption disabled, should not assert!
6087 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6088 * instead!!!
6089 */
6090DECLINLINE(int) hmR0VmxImportGuestRFlags(PVMCPU pVCpu)
6091{
6092 uint32_t u32Val;
6093 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6094 if (pCtx->fExtrn & CPUMCTX_EXTRN_RFLAGS)
6095 {
6096 int rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Val);
6097 if (RT_SUCCESS(rc))
6098 {
6099 pCtx->eflags.u32 = u32Val;
6100
6101 /* Restore eflags for real-on-v86-mode hack. */
6102 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6103 {
6104 pCtx->eflags.Bits.u1VM = 0;
6105 pCtx->eflags.Bits.u2IOPL = pVCpu->hm.s.vmx.RealMode.Eflags.Bits.u2IOPL;
6106 }
6107 }
6108 pCtx->fExtrn &= ~CPUMCTX_EXTRN_RFLAGS;
6109 return rc;
6110 }
6111 return VINF_SUCCESS;
6112}
6113
6114
6115/**
6116 * Imports the guest interruptibility-state from the VMCS back into the guest-CPU
6117 * context.
6118 *
6119 * @returns VBox status code.
6120 * @param pVCpu The cross context virtual CPU structure.
6121 *
6122 * @remarks Called with interrupts and/or preemption disabled, try not to assert and
6123 * do not log!
6124 * @remarks Do -not- call this function directly, use hmR0VmxImportGuestState()
6125 * instead!!!
6126 */
6127DECLINLINE(int) hmR0VmxImportGuestIntrState(PVMCPU pVCpu)
6128{
6129 uint32_t u32Val;
6130 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6131 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32Val);
6132 if (RT_SUCCESS(rc))
6133 {
6134 /*
6135 * We additionally have a requirement to import RIP, RFLAGS depending on whether we
6136 * might need them in hmR0VmxEvaluatePendingEvent().
6137 */
6138 if (!u32Val)
6139 {
6140 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6141 {
6142 rc = hmR0VmxImportGuestRip(pVCpu);
6143 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6144 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6145 }
6146
6147 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6148 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6149 }
6150 else
6151 {
6152 rc = hmR0VmxImportGuestRip(pVCpu);
6153 rc |= hmR0VmxImportGuestRFlags(pVCpu);
6154
6155 if (u32Val & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
6156 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI))
6157 {
6158 EMSetInhibitInterruptsPC(pVCpu, pCtx->rip);
6159 }
6160 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
6161 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
6162
6163 if (u32Val & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI)
6164 {
6165 if (!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6166 VMCPU_FF_SET(pVCpu, VMCPU_FF_BLOCK_NMIS);
6167 }
6168 else if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS))
6169 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_BLOCK_NMIS);
6170 }
6171 }
6172 return rc;
6173}
6174
6175
6176/**
6177 * Worker for VMXR0ImportStateOnDemand.
6178 *
6179 * @returns VBox status code.
6180 * @param pVCpu The cross context virtual CPU structure.
6181 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6182 */
6183static int hmR0VmxImportGuestState(PVMCPU pVCpu, uint64_t fWhat)
6184{
6185#define VMXLOCAL_BREAK_RC(a_rc) \
6186 if (RT_FAILURE(a_rc)) \
6187 break
6188
6189 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatImportGuestState, x);
6190
6191 int rc = VINF_SUCCESS;
6192 PVM pVM = pVCpu->CTX_SUFF(pVM);
6193 PCPUMCTX pCtx = &pVCpu->cpum.GstCtx;
6194 uint64_t u64Val;
6195 uint32_t u32Val;
6196
6197 Log4Func(("fExtrn=%#RX64 fWhat=%#RX64\n", pCtx->fExtrn, fWhat));
6198
6199 /*
6200 * We disable interrupts to make the updating of the state and in particular
6201 * the fExtrn modification atomic wrt to preemption hooks.
6202 */
6203 RTCCUINTREG const fEFlags = ASMIntDisableFlags();
6204
6205 fWhat &= pCtx->fExtrn;
6206 if (fWhat)
6207 {
6208 do
6209 {
6210 if (fWhat & CPUMCTX_EXTRN_RIP)
6211 {
6212 rc = hmR0VmxImportGuestRip(pVCpu);
6213 VMXLOCAL_BREAK_RC(rc);
6214 }
6215
6216 if (fWhat & CPUMCTX_EXTRN_RFLAGS)
6217 {
6218 rc = hmR0VmxImportGuestRFlags(pVCpu);
6219 VMXLOCAL_BREAK_RC(rc);
6220 }
6221
6222 if (fWhat & CPUMCTX_EXTRN_HM_VMX_INT_STATE)
6223 {
6224 rc = hmR0VmxImportGuestIntrState(pVCpu);
6225 VMXLOCAL_BREAK_RC(rc);
6226 }
6227
6228 if (fWhat & CPUMCTX_EXTRN_RSP)
6229 {
6230 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_RSP, &u64Val);
6231 VMXLOCAL_BREAK_RC(rc);
6232 pCtx->rsp = u64Val;
6233 }
6234
6235 if (fWhat & CPUMCTX_EXTRN_SREG_MASK)
6236 {
6237 if (fWhat & CPUMCTX_EXTRN_CS)
6238 {
6239 rc = HMVMX_IMPORT_SREG(CS, &pCtx->cs);
6240 VMXLOCAL_BREAK_RC(rc);
6241 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6242 pCtx->cs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrCS.u;
6243 }
6244 if (fWhat & CPUMCTX_EXTRN_SS)
6245 {
6246 rc = HMVMX_IMPORT_SREG(SS, &pCtx->ss);
6247 VMXLOCAL_BREAK_RC(rc);
6248 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6249 pCtx->ss.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrSS.u;
6250 }
6251 if (fWhat & CPUMCTX_EXTRN_DS)
6252 {
6253 rc = HMVMX_IMPORT_SREG(DS, &pCtx->ds);
6254 VMXLOCAL_BREAK_RC(rc);
6255 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6256 pCtx->ds.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrDS.u;
6257 }
6258 if (fWhat & CPUMCTX_EXTRN_ES)
6259 {
6260 rc = HMVMX_IMPORT_SREG(ES, &pCtx->es);
6261 VMXLOCAL_BREAK_RC(rc);
6262 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6263 pCtx->es.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrES.u;
6264 }
6265 if (fWhat & CPUMCTX_EXTRN_FS)
6266 {
6267 rc = HMVMX_IMPORT_SREG(FS, &pCtx->fs);
6268 VMXLOCAL_BREAK_RC(rc);
6269 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6270 pCtx->fs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrFS.u;
6271 }
6272 if (fWhat & CPUMCTX_EXTRN_GS)
6273 {
6274 rc = HMVMX_IMPORT_SREG(GS, &pCtx->gs);
6275 VMXLOCAL_BREAK_RC(rc);
6276 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6277 pCtx->gs.Attr.u = pVCpu->hm.s.vmx.RealMode.AttrGS.u;
6278 }
6279 }
6280
6281 if (fWhat & CPUMCTX_EXTRN_TABLE_MASK)
6282 {
6283 if (fWhat & CPUMCTX_EXTRN_LDTR)
6284 {
6285 rc = HMVMX_IMPORT_SREG(LDTR, &pCtx->ldtr);
6286 VMXLOCAL_BREAK_RC(rc);
6287 }
6288
6289 if (fWhat & CPUMCTX_EXTRN_GDTR)
6290 {
6291 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
6292 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
6293 VMXLOCAL_BREAK_RC(rc);
6294 pCtx->gdtr.pGdt = u64Val;
6295 pCtx->gdtr.cbGdt = u32Val;
6296 }
6297
6298 /* Guest IDTR. */
6299 if (fWhat & CPUMCTX_EXTRN_IDTR)
6300 {
6301 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
6302 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
6303 VMXLOCAL_BREAK_RC(rc);
6304 pCtx->idtr.pIdt = u64Val;
6305 pCtx->idtr.cbIdt = u32Val;
6306 }
6307
6308 /* Guest TR. */
6309 if (fWhat & CPUMCTX_EXTRN_TR)
6310 {
6311 /* Real-mode emulation using virtual-8086 mode has the fake TSS (pRealModeTSS) in TR, don't save that one. */
6312 if (!pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
6313 {
6314 rc = HMVMX_IMPORT_SREG(TR, &pCtx->tr);
6315 VMXLOCAL_BREAK_RC(rc);
6316 }
6317 }
6318 }
6319
6320 if (fWhat & CPUMCTX_EXTRN_SYSENTER_MSRS)
6321 {
6322 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_EIP, &pCtx->SysEnter.eip);
6323 rc |= VMXReadVmcsGstN(VMX_VMCS_GUEST_SYSENTER_ESP, &pCtx->SysEnter.esp);
6324 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_SYSENTER_CS, &u32Val);
6325 pCtx->SysEnter.cs = u32Val;
6326 VMXLOCAL_BREAK_RC(rc);
6327 }
6328
6329#if HC_ARCH_BITS == 64
6330 if (fWhat & CPUMCTX_EXTRN_KERNEL_GS_BASE)
6331 {
6332 if ( pVM->hm.s.fAllow64BitGuests
6333 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6334 pCtx->msrKERNELGSBASE = ASMRdMsr(MSR_K8_KERNEL_GS_BASE);
6335 }
6336
6337 if (fWhat & CPUMCTX_EXTRN_SYSCALL_MSRS)
6338 {
6339 if ( pVM->hm.s.fAllow64BitGuests
6340 && (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST))
6341 {
6342 pCtx->msrLSTAR = ASMRdMsr(MSR_K8_LSTAR);
6343 pCtx->msrSTAR = ASMRdMsr(MSR_K6_STAR);
6344 pCtx->msrSFMASK = ASMRdMsr(MSR_K8_SF_MASK);
6345 }
6346 }
6347#endif
6348
6349 if ( (fWhat & (CPUMCTX_EXTRN_TSC_AUX | CPUMCTX_EXTRN_OTHER_MSRS))
6350#if HC_ARCH_BITS == 32
6351 || (fWhat & (CPUMCTX_EXTRN_KERNEL_GS_BASE | CPUMCTX_EXTRN_SYSCALL_MSRS))
6352#endif
6353 )
6354 {
6355 PCVMXAUTOMSR pMsr = (PVMXAUTOMSR)pVCpu->hm.s.vmx.pvGuestMsr;
6356 uint32_t const cMsrs = pVCpu->hm.s.vmx.cMsrs;
6357 for (uint32_t i = 0; i < cMsrs; i++, pMsr++)
6358 {
6359 switch (pMsr->u32Msr)
6360 {
6361#if HC_ARCH_BITS == 32
6362 case MSR_K8_LSTAR: pCtx->msrLSTAR = pMsr->u64Value; break;
6363 case MSR_K6_STAR: pCtx->msrSTAR = pMsr->u64Value; break;
6364 case MSR_K8_SF_MASK: pCtx->msrSFMASK = pMsr->u64Value; break;
6365 case MSR_K8_KERNEL_GS_BASE: pCtx->msrKERNELGSBASE = pMsr->u64Value; break;
6366#endif
6367 case MSR_IA32_SPEC_CTRL: CPUMSetGuestSpecCtrl(pVCpu, pMsr->u64Value); break;
6368 case MSR_K8_TSC_AUX: CPUMSetGuestTscAux(pVCpu, pMsr->u64Value); break;
6369 case MSR_K6_EFER: /* EFER can't be changed without causing a VM-exit */ break;
6370 default:
6371 {
6372 pVCpu->hm.s.u32HMError = pMsr->u32Msr;
6373 ASMSetFlags(fEFlags);
6374 AssertMsgFailed(("Unexpected MSR in auto-load/store area. uMsr=%#RX32 cMsrs=%u\n", pMsr->u32Msr,
6375 cMsrs));
6376 return VERR_HM_UNEXPECTED_LD_ST_MSR;
6377 }
6378 }
6379 }
6380 }
6381
6382 if (fWhat & CPUMCTX_EXTRN_DR7)
6383 {
6384 if (!pVCpu->hm.s.fUsingHyperDR7)
6385 {
6386 /* Upper 32-bits are always zero. See Intel spec. 2.7.3 "Loading and Storing Debug Registers". */
6387 rc = VMXReadVmcs32(VMX_VMCS_GUEST_DR7, &u32Val);
6388 VMXLOCAL_BREAK_RC(rc);
6389 pCtx->dr[7] = u32Val;
6390 }
6391 }
6392
6393 if (fWhat & CPUMCTX_EXTRN_CR_MASK)
6394 {
6395 uint32_t u32Shadow;
6396 if (fWhat & CPUMCTX_EXTRN_CR0)
6397 {
6398 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val);
6399 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR0_READ_SHADOW, &u32Shadow);
6400 VMXLOCAL_BREAK_RC(rc);
6401 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR0Mask)
6402 | (u32Shadow & pVCpu->hm.s.vmx.u32CR0Mask);
6403 VMMRZCallRing3Disable(pVCpu); /* Calls into PGM which has Log statements. */
6404 CPUMSetGuestCR0(pVCpu, u32Val);
6405 VMMRZCallRing3Enable(pVCpu);
6406 }
6407
6408 if (fWhat & CPUMCTX_EXTRN_CR4)
6409 {
6410 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &u32Val);
6411 rc |= VMXReadVmcs32(VMX_VMCS_CTRL_CR4_READ_SHADOW, &u32Shadow);
6412 VMXLOCAL_BREAK_RC(rc);
6413 u32Val = (u32Val & ~pVCpu->hm.s.vmx.u32CR4Mask)
6414 | (u32Shadow & pVCpu->hm.s.vmx.u32CR4Mask);
6415 CPUMSetGuestCR4(pVCpu, u32Val);
6416 }
6417
6418 if (fWhat & CPUMCTX_EXTRN_CR3)
6419 {
6420 /* CR0.PG bit changes are always intercepted, so it's up to date. */
6421 if ( pVM->hm.s.vmx.fUnrestrictedGuest
6422 || ( pVM->hm.s.fNestedPaging
6423 && CPUMIsGuestPagingEnabledEx(pCtx)))
6424 {
6425 rc = VMXReadVmcsGstN(VMX_VMCS_GUEST_CR3, &u64Val);
6426 if (pCtx->cr3 != u64Val)
6427 {
6428 CPUMSetGuestCR3(pVCpu, u64Val);
6429 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_CR3);
6430 }
6431
6432 /* If the guest is in PAE mode, sync back the PDPE's into the guest state.
6433 Note: CR4.PAE, CR0.PG, EFER bit changes are always intercepted, so they're up to date. */
6434 if (CPUMIsGuestInPAEModeEx(pCtx))
6435 {
6436 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &pVCpu->hm.s.aPdpes[0].u);
6437 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &pVCpu->hm.s.aPdpes[1].u);
6438 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &pVCpu->hm.s.aPdpes[2].u);
6439 rc |= VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &pVCpu->hm.s.aPdpes[3].u);
6440 VMXLOCAL_BREAK_RC(rc);
6441 VMCPU_FF_SET(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES);
6442 }
6443 }
6444 }
6445 }
6446 } while (0);
6447
6448 if (RT_SUCCESS(rc))
6449 {
6450 /* Update fExtrn. */
6451 pCtx->fExtrn &= ~fWhat;
6452
6453 /* If everything has been imported, clear the HM keeper bit. */
6454 if (!(pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL))
6455 {
6456 pCtx->fExtrn &= ~CPUMCTX_EXTRN_KEEPER_HM;
6457 Assert(!pCtx->fExtrn);
6458 }
6459 }
6460 }
6461 else
6462 AssertMsg(!pCtx->fExtrn || (pCtx->fExtrn & HMVMX_CPUMCTX_EXTRN_ALL), ("%#RX64\n", pCtx->fExtrn));
6463
6464 ASMSetFlags(fEFlags);
6465
6466 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatImportGuestState, x);
6467
6468 /*
6469 * Honor any pending CR3 updates.
6470 *
6471 * Consider this scenario: VM-exit -> VMMRZCallRing3Enable() -> do stuff that causes a longjmp -> hmR0VmxCallRing3Callback()
6472 * -> VMMRZCallRing3Disable() -> hmR0VmxImportGuestState() -> Sets VMCPU_FF_HM_UPDATE_CR3 pending -> return from the longjmp
6473 * -> continue with VM-exit handling -> hmR0VmxImportGuestState() and here we are.
6474 *
6475 * The reason for such complicated handling is because VM-exits that call into PGM expect CR3 to be up-to-date and thus
6476 * if any CR3-saves -before- the VM-exit (longjmp) postponed the CR3 update via the force-flag, any VM-exit handler that
6477 * calls into PGM when it re-saves CR3 will end up here and we call PGMUpdateCR3(). This is why the code below should
6478 * -NOT- check if CPUMCTX_EXTRN_CR3 is set!
6479 *
6480 * The longjmp exit path can't check these CR3 force-flags and call code that takes a lock again. We cover for it here.
6481 */
6482 if (VMMRZCallRing3IsEnabled(pVCpu))
6483 {
6484 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
6485 {
6486 Assert(!(ASMAtomicUoReadU64(&pCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
6487 PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
6488 }
6489
6490 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
6491 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
6492
6493 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
6494 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
6495 }
6496
6497 return VINF_SUCCESS;
6498#undef VMXLOCAL_BREAK_RC
6499}
6500
6501
6502/**
6503 * Saves the guest state from the VMCS into the guest-CPU context.
6504 *
6505 * @returns VBox status code.
6506 * @param pVCpu The cross context virtual CPU structure.
6507 * @param fWhat What to import, CPUMCTX_EXTRN_XXX.
6508 */
6509VMMR0DECL(int) VMXR0ImportStateOnDemand(PVMCPU pVCpu, uint64_t fWhat)
6510{
6511 return hmR0VmxImportGuestState(pVCpu, fWhat);
6512}
6513
6514
6515/**
6516 * Check per-VM and per-VCPU force flag actions that require us to go back to
6517 * ring-3 for one reason or another.
6518 *
6519 * @returns Strict VBox status code (i.e. informational status codes too)
6520 * @retval VINF_SUCCESS if we don't have any actions that require going back to
6521 * ring-3.
6522 * @retval VINF_PGM_SYNC_CR3 if we have pending PGM CR3 sync.
6523 * @retval VINF_EM_PENDING_REQUEST if we have pending requests (like hardware
6524 * interrupts)
6525 * @retval VINF_PGM_POOL_FLUSH_PENDING if PGM is doing a pool flush and requires
6526 * all EMTs to be in ring-3.
6527 * @retval VINF_EM_RAW_TO_R3 if there is pending DMA requests.
6528 * @retval VINF_EM_NO_MEMORY PGM is out of memory, we need to return
6529 * to the EM loop.
6530 *
6531 * @param pVCpu The cross context virtual CPU structure.
6532 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6533 * out-of-sync. Make sure to update the required fields
6534 * before using them.
6535 * @param fStepping Running in hmR0VmxRunGuestCodeStep().
6536 */
6537static VBOXSTRICTRC hmR0VmxCheckForceFlags(PVMCPU pVCpu, PCPUMCTX pMixedCtx, bool fStepping)
6538{
6539 Assert(VMMRZCallRing3IsEnabled(pVCpu));
6540
6541 /*
6542 * Anything pending? Should be more likely than not if we're doing a good job.
6543 */
6544 PVM pVM = pVCpu->CTX_SUFF(pVM);
6545 if ( !fStepping
6546 ? !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_MASK)
6547 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_MASK)
6548 : !VM_FF_IS_PENDING(pVM, VM_FF_HP_R0_PRE_HM_STEP_MASK)
6549 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HP_R0_PRE_HM_STEP_MASK) )
6550 return VINF_SUCCESS;
6551
6552 /* Pending PGM C3 sync. */
6553 if (VMCPU_FF_IS_PENDING(pVCpu,VMCPU_FF_PGM_SYNC_CR3 | VMCPU_FF_PGM_SYNC_CR3_NON_GLOBAL))
6554 {
6555 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & (CPUMCTX_EXTRN_CR0 | CPUMCTX_EXTRN_CR3 | CPUMCTX_EXTRN_CR4)));
6556 VBOXSTRICTRC rcStrict2 = PGMSyncCR3(pVCpu, pMixedCtx->cr0, pMixedCtx->cr3, pMixedCtx->cr4,
6557 VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_PGM_SYNC_CR3));
6558 if (rcStrict2 != VINF_SUCCESS)
6559 {
6560 AssertRC(VBOXSTRICTRC_VAL(rcStrict2));
6561 Log4Func(("PGMSyncCR3 forcing us back to ring-3. rc2=%d\n", VBOXSTRICTRC_VAL(rcStrict2)));
6562 return rcStrict2;
6563 }
6564 }
6565
6566 /* Pending HM-to-R3 operations (critsects, timers, EMT rendezvous etc.) */
6567 if ( VM_FF_IS_PENDING(pVM, VM_FF_HM_TO_R3_MASK)
6568 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
6569 {
6570 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
6571 int rc2 = RT_UNLIKELY(VM_FF_IS_PENDING(pVM, VM_FF_PGM_NO_MEMORY)) ? VINF_EM_NO_MEMORY : VINF_EM_RAW_TO_R3;
6572 Log4Func(("HM_TO_R3 forcing us back to ring-3. rc=%d\n", rc2));
6573 return rc2;
6574 }
6575
6576 /* Pending VM request packets, such as hardware interrupts. */
6577 if ( VM_FF_IS_PENDING(pVM, VM_FF_REQUEST)
6578 || VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_REQUEST))
6579 {
6580 Log4Func(("Pending VM request forcing us back to ring-3\n"));
6581 return VINF_EM_PENDING_REQUEST;
6582 }
6583
6584 /* Pending PGM pool flushes. */
6585 if (VM_FF_IS_PENDING(pVM, VM_FF_PGM_POOL_FLUSH_PENDING))
6586 {
6587 Log4Func(("PGM pool flush pending forcing us back to ring-3\n"));
6588 return VINF_PGM_POOL_FLUSH_PENDING;
6589 }
6590
6591 /* Pending DMA requests. */
6592 if (VM_FF_IS_PENDING(pVM, VM_FF_PDM_DMA))
6593 {
6594 Log4Func(("Pending DMA request forcing us back to ring-3\n"));
6595 return VINF_EM_RAW_TO_R3;
6596 }
6597
6598 return VINF_SUCCESS;
6599}
6600
6601
6602/**
6603 * Converts any TRPM trap into a pending HM event. This is typically used when
6604 * entering from ring-3 (not longjmp returns).
6605 *
6606 * @param pVCpu The cross context virtual CPU structure.
6607 */
6608static void hmR0VmxTrpmTrapToPendingEvent(PVMCPU pVCpu)
6609{
6610 Assert(TRPMHasTrap(pVCpu));
6611 Assert(!pVCpu->hm.s.Event.fPending);
6612
6613 uint8_t uVector;
6614 TRPMEVENT enmTrpmEvent;
6615 RTGCUINT uErrCode;
6616 RTGCUINTPTR GCPtrFaultAddress;
6617 uint8_t cbInstr;
6618
6619 int rc = TRPMQueryTrapAll(pVCpu, &uVector, &enmTrpmEvent, &uErrCode, &GCPtrFaultAddress, &cbInstr);
6620 AssertRC(rc);
6621
6622 /* Refer Intel spec. 24.8.3 "VM-entry Controls for Event Injection" for the format of u32IntInfo. */
6623 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
6624 if (enmTrpmEvent == TRPM_TRAP)
6625 {
6626 switch (uVector)
6627 {
6628 case X86_XCPT_NMI:
6629 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6630 break;
6631
6632 case X86_XCPT_BP:
6633 case X86_XCPT_OF:
6634 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6635 break;
6636
6637 case X86_XCPT_PF:
6638 case X86_XCPT_DF:
6639 case X86_XCPT_TS:
6640 case X86_XCPT_NP:
6641 case X86_XCPT_SS:
6642 case X86_XCPT_GP:
6643 case X86_XCPT_AC:
6644 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
6645 RT_FALL_THRU();
6646 default:
6647 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6648 break;
6649 }
6650 }
6651 else if (enmTrpmEvent == TRPM_HARDWARE_INT)
6652 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6653 else if (enmTrpmEvent == TRPM_SOFTWARE_INT)
6654 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
6655 else
6656 AssertMsgFailed(("Invalid TRPM event type %d\n", enmTrpmEvent));
6657
6658 rc = TRPMResetTrap(pVCpu);
6659 AssertRC(rc);
6660 Log4(("TRPM->HM event: u32IntInfo=%#RX32 enmTrpmEvent=%d cbInstr=%u uErrCode=%#RX32 GCPtrFaultAddress=%#RGv\n",
6661 u32IntInfo, enmTrpmEvent, cbInstr, uErrCode, GCPtrFaultAddress));
6662
6663 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, uErrCode, GCPtrFaultAddress);
6664}
6665
6666
6667/**
6668 * Converts the pending HM event into a TRPM trap.
6669 *
6670 * @param pVCpu The cross context virtual CPU structure.
6671 */
6672static void hmR0VmxPendingEventToTrpmTrap(PVMCPU pVCpu)
6673{
6674 Assert(pVCpu->hm.s.Event.fPending);
6675
6676 uint32_t uVectorType = VMX_IDT_VECTORING_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
6677 uint32_t uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVCpu->hm.s.Event.u64IntInfo);
6678 bool fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVCpu->hm.s.Event.u64IntInfo);
6679 uint32_t uErrorCode = pVCpu->hm.s.Event.u32ErrCode;
6680
6681 /* If a trap was already pending, we did something wrong! */
6682 Assert(TRPMQueryTrap(pVCpu, NULL /* pu8TrapNo */, NULL /* pEnmType */) == VERR_TRPM_NO_ACTIVE_TRAP);
6683
6684 TRPMEVENT enmTrapType;
6685 switch (uVectorType)
6686 {
6687 case VMX_IDT_VECTORING_INFO_TYPE_EXT_INT:
6688 enmTrapType = TRPM_HARDWARE_INT;
6689 break;
6690
6691 case VMX_IDT_VECTORING_INFO_TYPE_SW_INT:
6692 enmTrapType = TRPM_SOFTWARE_INT;
6693 break;
6694
6695 case VMX_IDT_VECTORING_INFO_TYPE_NMI:
6696 case VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT:
6697 case VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT: /* #BP and #OF */
6698 case VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT:
6699 enmTrapType = TRPM_TRAP;
6700 break;
6701
6702 default:
6703 AssertMsgFailed(("Invalid trap type %#x\n", uVectorType));
6704 enmTrapType = TRPM_32BIT_HACK;
6705 break;
6706 }
6707
6708 Log4(("HM event->TRPM: uVector=%#x enmTrapType=%d\n", uVector, enmTrapType));
6709
6710 int rc = TRPMAssertTrap(pVCpu, uVector, enmTrapType);
6711 AssertRC(rc);
6712
6713 if (fErrorCodeValid)
6714 TRPMSetErrorCode(pVCpu, uErrorCode);
6715
6716 if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
6717 && uVector == X86_XCPT_PF)
6718 {
6719 TRPMSetFaultAddress(pVCpu, pVCpu->hm.s.Event.GCPtrFaultAddress);
6720 }
6721 else if ( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6722 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_XCPT
6723 || uVectorType == VMX_IDT_VECTORING_INFO_TYPE_PRIV_SW_XCPT)
6724 {
6725 AssertMsg( uVectorType == VMX_IDT_VECTORING_INFO_TYPE_SW_INT
6726 || (uVector == X86_XCPT_BP || uVector == X86_XCPT_OF),
6727 ("Invalid vector: uVector=%#x uVectorType=%#x\n", uVector, uVectorType));
6728 TRPMSetInstrLength(pVCpu, pVCpu->hm.s.Event.cbInstr);
6729 }
6730
6731 /* Clear any pending events from the VMCS. */
6732 rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, 0); AssertRC(rc);
6733 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, 0); AssertRC(rc);
6734
6735 /* We're now done converting the pending event. */
6736 pVCpu->hm.s.Event.fPending = false;
6737}
6738
6739
6740/**
6741 * Does the necessary state syncing before returning to ring-3 for any reason
6742 * (longjmp, preemption, voluntary exits to ring-3) from VT-x.
6743 *
6744 * @returns VBox status code.
6745 * @param pVCpu The cross context virtual CPU structure.
6746 * @param fImportState Whether to import the guest state from the VMCS back
6747 * to the guest-CPU context.
6748 *
6749 * @remarks No-long-jmp zone!!!
6750 */
6751static int hmR0VmxLeave(PVMCPU pVCpu, bool fImportState)
6752{
6753 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6754 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6755
6756 RTCPUID idCpu = RTMpCpuId();
6757 Log4Func(("HostCpuId=%u\n", idCpu));
6758
6759 /*
6760 * !!! IMPORTANT !!!
6761 * If you modify code here, check whether hmR0VmxCallRing3Callback() needs to be updated too.
6762 */
6763
6764 /* Save the guest state if necessary. */
6765 if (fImportState)
6766 {
6767 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
6768 AssertRCReturn(rc, rc);
6769 }
6770
6771 /* Restore host FPU state if necessary. We will resync on next R0 reentry. */
6772 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
6773 Assert(!CPUMIsGuestFPUStateActive(pVCpu));
6774
6775 /* Restore host debug registers if necessary. We will resync on next R0 reentry. */
6776#ifdef VBOX_STRICT
6777 if (CPUMIsHyperDebugStateActive(pVCpu))
6778 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT);
6779#endif
6780 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
6781 Assert(!CPUMIsGuestDebugStateActive(pVCpu) && !CPUMIsGuestDebugStateActivePending(pVCpu));
6782 Assert(!CPUMIsHyperDebugStateActive(pVCpu) && !CPUMIsHyperDebugStateActivePending(pVCpu));
6783
6784#if HC_ARCH_BITS == 64
6785 /* Restore host-state bits that VT-x only restores partially. */
6786 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
6787 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
6788 {
6789 Log4Func(("Restoring Host State: fRestoreHostFlags=%#RX32 HostCpuId=%u\n", pVCpu->hm.s.vmx.fRestoreHostFlags, idCpu));
6790 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
6791 }
6792 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
6793#endif
6794
6795 /* Restore the lazy host MSRs as we're leaving VT-x context. */
6796 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
6797 {
6798 /* We shouldn't restore the host MSRs without saving the guest MSRs first. */
6799 if (!fImportState)
6800 {
6801 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_KERNEL_GS_BASE
6802 | CPUMCTX_EXTRN_SYSCALL_MSRS);
6803 AssertRCReturn(rc, rc);
6804 }
6805 hmR0VmxLazyRestoreHostMsrs(pVCpu);
6806 Assert(!pVCpu->hm.s.vmx.fLazyMsrs);
6807 }
6808 else
6809 pVCpu->hm.s.vmx.fLazyMsrs = 0;
6810
6811 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
6812 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
6813
6814 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatEntry);
6815 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatImportGuestState);
6816 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExportGuestState);
6817 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatPreExit);
6818 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitHandling);
6819 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitIO);
6820 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitMovCRx);
6821 STAM_PROFILE_ADV_SET_STOPPED(&pVCpu->hm.s.StatExitXcptNmi);
6822 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6823
6824 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
6825
6826 /** @todo This partially defeats the purpose of having preemption hooks.
6827 * The problem is, deregistering the hooks should be moved to a place that
6828 * lasts until the EMT is about to be destroyed not everytime while leaving HM
6829 * context.
6830 */
6831 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
6832 {
6833 int rc = VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
6834 AssertRCReturn(rc, rc);
6835
6836 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
6837 Log4Func(("Cleared Vmcs. HostCpuId=%u\n", idCpu));
6838 }
6839 Assert(!(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_LAUNCHED));
6840 NOREF(idCpu);
6841
6842 return VINF_SUCCESS;
6843}
6844
6845
6846/**
6847 * Leaves the VT-x session.
6848 *
6849 * @returns VBox status code.
6850 * @param pVCpu The cross context virtual CPU structure.
6851 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6852 * out-of-sync. Make sure to update the required fields
6853 * before using them.
6854 *
6855 * @remarks No-long-jmp zone!!!
6856 */
6857static int hmR0VmxLeaveSession(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6858{
6859 HM_DISABLE_PREEMPT();
6860 HMVMX_ASSERT_CPU_SAFE();
6861 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
6862 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
6863
6864 /* When thread-context hooks are used, we can avoid doing the leave again if we had been preempted before
6865 and done this from the VMXR0ThreadCtxCallback(). */
6866 if (!pVCpu->hm.s.fLeaveDone)
6867 {
6868 int rc2 = hmR0VmxLeave(pVCpu, true /* fImportState */);
6869 AssertRCReturnStmt(rc2, HM_RESTORE_PREEMPT(), rc2);
6870 pVCpu->hm.s.fLeaveDone = true;
6871 }
6872 Assert(!pMixedCtx->fExtrn); NOREF(pMixedCtx);
6873
6874 /*
6875 * !!! IMPORTANT !!!
6876 * If you modify code here, make sure to check whether hmR0VmxCallRing3Callback() needs to be updated too.
6877 */
6878
6879 /* Deregister hook now that we've left HM context before re-enabling preemption. */
6880 /** @todo Deregistering here means we need to VMCLEAR always
6881 * (longjmp/exit-to-r3) in VT-x which is not efficient. */
6882 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
6883 VMMR0ThreadCtxHookDisable(pVCpu);
6884
6885 /* Leave HM context. This takes care of local init (term). */
6886 int rc = HMR0LeaveCpu(pVCpu);
6887
6888 HM_RESTORE_PREEMPT();
6889 return rc;
6890}
6891
6892
6893/**
6894 * Does the necessary state syncing before doing a longjmp to ring-3.
6895 *
6896 * @returns VBox status code.
6897 * @param pVCpu The cross context virtual CPU structure.
6898 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6899 * out-of-sync. Make sure to update the required fields
6900 * before using them.
6901 *
6902 * @remarks No-long-jmp zone!!!
6903 */
6904DECLINLINE(int) hmR0VmxLongJmpToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
6905{
6906 return hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6907}
6908
6909
6910/**
6911 * Take necessary actions before going back to ring-3.
6912 *
6913 * An action requires us to go back to ring-3. This function does the necessary
6914 * steps before we can safely return to ring-3. This is not the same as longjmps
6915 * to ring-3, this is voluntary and prepares the guest so it may continue
6916 * executing outside HM (recompiler/IEM).
6917 *
6918 * @returns VBox status code.
6919 * @param pVCpu The cross context virtual CPU structure.
6920 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
6921 * out-of-sync. Make sure to update the required fields
6922 * before using them.
6923 * @param rcExit The reason for exiting to ring-3. Can be
6924 * VINF_VMM_UNKNOWN_RING3_CALL.
6925 */
6926static int hmR0VmxExitToRing3(PVMCPU pVCpu, PCPUMCTX pMixedCtx, VBOXSTRICTRC rcExit)
6927{
6928 Assert(pVCpu);
6929 Assert(pMixedCtx);
6930 HMVMX_ASSERT_PREEMPT_SAFE();
6931
6932 if (RT_UNLIKELY(rcExit == VERR_VMX_INVALID_VMCS_PTR))
6933 {
6934 VMXGetActivatedVmcs(&pVCpu->hm.s.vmx.LastError.u64VMCSPhys);
6935 pVCpu->hm.s.vmx.LastError.u32VMCSRevision = *(uint32_t *)pVCpu->hm.s.vmx.pvVmcs;
6936 pVCpu->hm.s.vmx.LastError.idEnteredCpu = pVCpu->hm.s.idEnteredCpu;
6937 /* LastError.idCurrentCpu was updated in hmR0VmxPreRunGuestCommitted(). */
6938 }
6939
6940 /* Please, no longjumps here (any logging shouldn't flush jump back to ring-3). NO LOGGING BEFORE THIS POINT! */
6941 VMMRZCallRing3Disable(pVCpu);
6942 Log4Func(("rcExit=%d\n", VBOXSTRICTRC_VAL(rcExit)));
6943
6944 /* We need to do this only while truly exiting the "inner loop" back to ring-3 and -not- for any longjmp to ring3. */
6945 if (pVCpu->hm.s.Event.fPending)
6946 {
6947 hmR0VmxPendingEventToTrpmTrap(pVCpu);
6948 Assert(!pVCpu->hm.s.Event.fPending);
6949 }
6950
6951 /* Clear interrupt-window and NMI-window controls as we re-evaluate it when we return from ring-3. */
6952 hmR0VmxClearIntNmiWindowsVmcs(pVCpu);
6953
6954 /* If we're emulating an instruction, we shouldn't have any TRPM traps pending
6955 and if we're injecting an event we should have a TRPM trap pending. */
6956 AssertMsg(rcExit != VINF_EM_RAW_INJECT_TRPM_EVENT || TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6957#ifndef DEBUG_bird /* Triggered after firing an NMI against NT4SP1, possibly a triple fault in progress. */
6958 AssertMsg(rcExit != VINF_EM_RAW_EMULATE_INSTR || !TRPMHasTrap(pVCpu), ("%Rrc\n", VBOXSTRICTRC_VAL(rcExit)));
6959#endif
6960
6961 /* Save guest state and restore host state bits. */
6962 int rc = hmR0VmxLeaveSession(pVCpu, pMixedCtx);
6963 AssertRCReturn(rc, rc);
6964 STAM_COUNTER_DEC(&pVCpu->hm.s.StatSwitchLongJmpToR3);
6965 /* Thread-context hooks are unregistered at this point!!! */
6966
6967 /* Sync recompiler state. */
6968 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_TO_R3);
6969 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_SYSENTER_MSR
6970 | CPUM_CHANGED_LDTR
6971 | CPUM_CHANGED_GDTR
6972 | CPUM_CHANGED_IDTR
6973 | CPUM_CHANGED_TR
6974 | CPUM_CHANGED_HIDDEN_SEL_REGS);
6975 if ( pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging
6976 && CPUMIsGuestPagingEnabledEx(pMixedCtx))
6977 {
6978 CPUMSetChangedFlags(pVCpu, CPUM_CHANGED_GLOBAL_TLB_FLUSH);
6979 }
6980
6981 Assert(!pVCpu->hm.s.fClearTrapFlag);
6982
6983 /* Update the exit-to-ring 3 reason. */
6984 pVCpu->hm.s.rcLastExitToR3 = VBOXSTRICTRC_VAL(rcExit);
6985
6986 /* On our way back from ring-3 reload the guest state if there is a possibility of it being changed. */
6987 if (rcExit != VINF_EM_RAW_INTERRUPT)
6988 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
6989
6990 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchExitToR3);
6991
6992 /* We do -not- want any longjmp notifications after this! We must return to ring-3 ASAP. */
6993 VMMRZCallRing3RemoveNotification(pVCpu);
6994 VMMRZCallRing3Enable(pVCpu);
6995
6996 return rc;
6997}
6998
6999
7000/**
7001 * VMMRZCallRing3() callback wrapper which saves the guest state before we
7002 * longjump to ring-3 and possibly get preempted.
7003 *
7004 * @returns VBox status code.
7005 * @param pVCpu The cross context virtual CPU structure.
7006 * @param enmOperation The operation causing the ring-3 longjump.
7007 * @param pvUser Opaque pointer to the guest-CPU context. The data
7008 * may be out-of-sync. Make sure to update the required
7009 * fields before using them.
7010 */
7011static DECLCALLBACK(int) hmR0VmxCallRing3Callback(PVMCPU pVCpu, VMMCALLRING3 enmOperation, void *pvUser)
7012{
7013 if (enmOperation == VMMCALLRING3_VM_R0_ASSERTION)
7014 {
7015 /*
7016 * !!! IMPORTANT !!!
7017 * If you modify code here, check whether hmR0VmxLeave() and hmR0VmxLeaveSession() needs to be updated too.
7018 * This is a stripped down version which gets out ASAP, trying to not trigger any further assertions.
7019 */
7020 VMMRZCallRing3RemoveNotification(pVCpu);
7021 VMMRZCallRing3Disable(pVCpu);
7022 RTTHREADPREEMPTSTATE PreemptState = RTTHREADPREEMPTSTATE_INITIALIZER;
7023 RTThreadPreemptDisable(&PreemptState);
7024
7025 CPUMR0FpuStateMaybeSaveGuestAndRestoreHost(pVCpu);
7026 CPUMR0DebugStateMaybeSaveGuestAndRestoreHost(pVCpu, true /* save DR6 */);
7027
7028#if HC_ARCH_BITS == 64
7029 /* Restore host-state bits that VT-x only restores partially. */
7030 if ( (pVCpu->hm.s.vmx.fRestoreHostFlags & VMX_RESTORE_HOST_REQUIRED)
7031 && (pVCpu->hm.s.vmx.fRestoreHostFlags & ~VMX_RESTORE_HOST_REQUIRED))
7032 VMXRestoreHostState(pVCpu->hm.s.vmx.fRestoreHostFlags, &pVCpu->hm.s.vmx.RestoreHost);
7033 pVCpu->hm.s.vmx.fRestoreHostFlags = 0;
7034#endif
7035
7036 /* Restore the lazy host MSRs as we're leaving VT-x context. */
7037 if (pVCpu->hm.s.vmx.fLazyMsrs & VMX_LAZY_MSRS_LOADED_GUEST)
7038 hmR0VmxLazyRestoreHostMsrs(pVCpu);
7039
7040 /* Update auto-load/store host MSRs values when we re-enter VT-x (as we could be on a different CPU). */
7041 pVCpu->hm.s.vmx.fUpdatedHostMsrs = false;
7042 VMCPU_CMPXCHG_STATE(pVCpu, VMCPUSTATE_STARTED_HM, VMCPUSTATE_STARTED_EXEC);
7043 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_ACTIVE)
7044 {
7045 VMXClearVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7046 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_CLEAR;
7047 }
7048
7049 /** @todo eliminate the need for calling VMMR0ThreadCtxHookDisable here! */
7050 VMMR0ThreadCtxHookDisable(pVCpu);
7051 HMR0LeaveCpu(pVCpu);
7052 RTThreadPreemptRestore(&PreemptState);
7053 return VINF_SUCCESS;
7054 }
7055
7056 Assert(pVCpu);
7057 Assert(pvUser);
7058 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7059 HMVMX_ASSERT_PREEMPT_SAFE();
7060
7061 VMMRZCallRing3Disable(pVCpu);
7062 Assert(VMMR0IsLogFlushDisabled(pVCpu));
7063
7064 Log4Func((" -> hmR0VmxLongJmpToRing3 enmOperation=%d\n", enmOperation));
7065
7066 int rc = hmR0VmxLongJmpToRing3(pVCpu, (PCPUMCTX)pvUser);
7067 AssertRCReturn(rc, rc);
7068
7069 VMMRZCallRing3Enable(pVCpu);
7070 return VINF_SUCCESS;
7071}
7072
7073
7074/**
7075 * Sets the interrupt-window exiting control in the VMCS which instructs VT-x to
7076 * cause a VM-exit as soon as the guest is in a state to receive interrupts.
7077 *
7078 * @param pVCpu The cross context virtual CPU structure.
7079 */
7080DECLINLINE(void) hmR0VmxSetIntWindowExitVmcs(PVMCPU pVCpu)
7081{
7082 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7083 {
7084 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT))
7085 {
7086 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7087 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7088 AssertRC(rc);
7089 Log4Func(("Setup interrupt-window exiting\n"));
7090 }
7091 } /* else we will deliver interrupts whenever the guest exits next and is in a state to receive events. */
7092}
7093
7094
7095/**
7096 * Clears the interrupt-window exiting control in the VMCS.
7097 *
7098 * @param pVCpu The cross context virtual CPU structure.
7099 */
7100DECLINLINE(void) hmR0VmxClearIntWindowExitVmcs(PVMCPU pVCpu)
7101{
7102 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT);
7103 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT;
7104 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7105 AssertRC(rc);
7106 Log4Func(("Cleared interrupt-window exiting\n"));
7107}
7108
7109
7110/**
7111 * Sets the NMI-window exiting control in the VMCS which instructs VT-x to
7112 * cause a VM-exit as soon as the guest is in a state to receive NMIs.
7113 *
7114 * @param pVCpu The cross context virtual CPU structure.
7115 */
7116DECLINLINE(void) hmR0VmxSetNmiWindowExitVmcs(PVMCPU pVCpu)
7117{
7118 if (RT_LIKELY(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7119 {
7120 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT))
7121 {
7122 pVCpu->hm.s.vmx.u32ProcCtls |= VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7123 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7124 AssertRC(rc);
7125 Log4Func(("Setup NMI-window exiting\n"));
7126 }
7127 } /* else we will deliver NMIs whenever we VM-exit next, even possibly nesting NMIs. Can't be helped on ancient CPUs. */
7128}
7129
7130
7131/**
7132 * Clears the NMI-window exiting control in the VMCS.
7133 *
7134 * @param pVCpu The cross context virtual CPU structure.
7135 */
7136DECLINLINE(void) hmR0VmxClearNmiWindowExitVmcs(PVMCPU pVCpu)
7137{
7138 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT);
7139 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT;
7140 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
7141 AssertRC(rc);
7142 Log4Func(("Cleared NMI-window exiting\n"));
7143}
7144
7145
7146/**
7147 * Evaluates the event to be delivered to the guest and sets it as the pending
7148 * event.
7149 *
7150 * @returns The VT-x guest-interruptibility state.
7151 * @param pVCpu The cross context virtual CPU structure.
7152 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7153 * out-of-sync. Make sure to update the required fields
7154 * before using them.
7155 */
7156static uint32_t hmR0VmxEvaluatePendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7157{
7158 /* Get the current interruptibility-state of the guest and then figure out what can be injected. */
7159 uint32_t const fIntrState = hmR0VmxGetGuestIntrState(pVCpu, pMixedCtx);
7160 bool const fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7161 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7162 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7163
7164 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7165 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7166 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7167 Assert(!TRPMHasTrap(pVCpu));
7168
7169 if (VMCPU_FF_TEST_AND_CLEAR(pVCpu, VMCPU_FF_UPDATE_APIC))
7170 APICUpdatePendingInterrupts(pVCpu);
7171
7172 /*
7173 * Toggling of interrupt force-flags here is safe since we update TRPM on premature exits
7174 * to ring-3 before executing guest code, see hmR0VmxExitToRing3(). We must NOT restore these force-flags.
7175 */
7176 /** @todo SMI. SMIs take priority over NMIs. */
7177 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INTERRUPT_NMI)) /* NMI. NMIs take priority over regular interrupts. */
7178 {
7179 /* On some CPUs block-by-STI also blocks NMIs. See Intel spec. 26.3.1.5 "Checks On Guest Non-Register State". */
7180 if ( !pVCpu->hm.s.Event.fPending
7181 && !fBlockNmi
7182 && !fBlockSti
7183 && !fBlockMovSS)
7184 {
7185 Log4Func(("Pending NMI\n"));
7186 uint32_t u32IntInfo = X86_XCPT_NMI | VMX_EXIT_INTERRUPTION_INFO_VALID;
7187 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7188
7189 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7190 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INTERRUPT_NMI);
7191 }
7192 else
7193 hmR0VmxSetNmiWindowExitVmcs(pVCpu);
7194 }
7195 /*
7196 * Check if the guest can receive external interrupts (PIC/APIC). Once PDMGetInterrupt() returns
7197 * a valid interrupt we must- deliver the interrupt. We can no longer re-request it from the APIC.
7198 */
7199 else if ( VMCPU_FF_IS_PENDING(pVCpu, (VMCPU_FF_INTERRUPT_APIC | VMCPU_FF_INTERRUPT_PIC))
7200 && !pVCpu->hm.s.fSingleInstruction)
7201 {
7202 Assert(!DBGFIsStepping(pVCpu));
7203 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7204 AssertRCReturn(rc, 0);
7205 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7206 if ( !pVCpu->hm.s.Event.fPending
7207 && !fBlockInt
7208 && !fBlockSti
7209 && !fBlockMovSS)
7210 {
7211 uint8_t u8Interrupt;
7212 rc = PDMGetInterrupt(pVCpu, &u8Interrupt);
7213 if (RT_SUCCESS(rc))
7214 {
7215 Log4Func(("Pending external interrupt u8Interrupt=%#x\n", u8Interrupt));
7216 uint32_t u32IntInfo = u8Interrupt
7217 | VMX_EXIT_INTERRUPTION_INFO_VALID
7218 | (VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7219
7220 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrfaultAddress */);
7221 }
7222 else if (rc == VERR_APIC_INTR_MASKED_BY_TPR)
7223 {
7224 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
7225 hmR0VmxApicSetTprThreshold(pVCpu, u8Interrupt >> 4);
7226 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchTprMaskedIrq);
7227
7228 /*
7229 * If the CPU doesn't have TPR shadowing, we will always get a VM-exit on TPR changes and
7230 * APICSetTpr() will end up setting the VMCPU_FF_INTERRUPT_APIC if required, so there is no
7231 * need to re-set this force-flag here.
7232 */
7233 }
7234 else
7235 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchGuestIrq);
7236 }
7237 else
7238 hmR0VmxSetIntWindowExitVmcs(pVCpu);
7239 }
7240
7241 return fIntrState;
7242}
7243
7244
7245/**
7246 * Sets a pending-debug exception to be delivered to the guest if the guest is
7247 * single-stepping in the VMCS.
7248 *
7249 * @param pVCpu The cross context virtual CPU structure.
7250 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7251 * out-of-sync. Make sure to update the required fields
7252 * before using them.
7253 */
7254DECLINLINE(int) hmR0VmxSetPendingDebugXcptVmcs(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7255{
7256 RT_NOREF(pVCpu);
7257 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS)); NOREF(pMixedCtx);
7258 return VMXWriteVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, VMX_VMCS_GUEST_DEBUG_EXCEPTIONS_BS);
7259}
7260
7261
7262/**
7263 * Injects any pending events into the guest if the guest is in a state to
7264 * receive them.
7265 *
7266 * @returns Strict VBox status code (i.e. informational status codes too).
7267 * @param pVCpu The cross context virtual CPU structure.
7268 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7269 * out-of-sync. Make sure to update the required fields
7270 * before using them.
7271 * @param fIntrState The VT-x guest-interruptibility state.
7272 * @param fStepping Running in hmR0VmxRunGuestCodeStep() and we should
7273 * return VINF_EM_DBG_STEPPED if the event was
7274 * dispatched directly.
7275 */
7276static VBOXSTRICTRC hmR0VmxInjectPendingEvent(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t fIntrState, bool fStepping)
7277{
7278 HMVMX_ASSERT_PREEMPT_SAFE();
7279 Assert(VMMRZCallRing3IsEnabled(pVCpu));
7280
7281 bool fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7282 bool fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7283
7284 Assert(!fBlockSti || !(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_RFLAGS));
7285 Assert(!(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI)); /* We don't support block-by-SMI yet.*/
7286 Assert(!fBlockSti || pMixedCtx->eflags.Bits.u1IF); /* Cannot set block-by-STI when interrupts are disabled. */
7287 Assert(!TRPMHasTrap(pVCpu));
7288
7289 VBOXSTRICTRC rcStrict = VINF_SUCCESS;
7290 if (pVCpu->hm.s.Event.fPending)
7291 {
7292 /*
7293 * Do -not- clear any interrupt-window exiting control here. We might have an interrupt
7294 * pending even while injecting an event and in this case, we want a VM-exit as soon as
7295 * the guest is ready for the next interrupt, see @bugref{6208#c45}.
7296 *
7297 * See Intel spec. 26.6.5 "Interrupt-Window Exiting and Virtual-Interrupt Delivery".
7298 */
7299 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVCpu->hm.s.Event.u64IntInfo);
7300#ifdef VBOX_STRICT
7301 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7302 {
7303 bool const fBlockInt = !(pMixedCtx->eflags.u32 & X86_EFL_IF);
7304 Assert(!fBlockInt);
7305 Assert(!fBlockSti);
7306 Assert(!fBlockMovSS);
7307 }
7308 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
7309 {
7310 bool const fBlockNmi = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI);
7311 Assert(!fBlockSti);
7312 Assert(!fBlockMovSS);
7313 Assert(!fBlockNmi);
7314 }
7315#endif
7316 Log4(("Injecting pending event vcpu[%RU32] u64IntInfo=%#RX64 Type=%#RX32\n", pVCpu->idCpu, pVCpu->hm.s.Event.u64IntInfo,
7317 uIntType));
7318 rcStrict = hmR0VmxInjectEventVmcs(pVCpu, pVCpu->hm.s.Event.u64IntInfo, pVCpu->hm.s.Event.cbInstr,
7319 pVCpu->hm.s.Event.u32ErrCode, pVCpu->hm.s.Event.GCPtrFaultAddress, fStepping,
7320 &fIntrState);
7321 AssertRCReturn(VBOXSTRICTRC_VAL(rcStrict), rcStrict);
7322
7323 /* Update the interruptibility-state as it could have been changed by
7324 hmR0VmxInjectEventVmcs() (e.g. real-on-v86 guest injecting software interrupts) */
7325 fBlockMovSS = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS);
7326 fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
7327
7328 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
7329 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectInterrupt);
7330 else
7331 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectXcpt);
7332 }
7333
7334 /* Deliver pending debug exception if the guest is single-stepping. Evaluate and set the BS bit. */
7335 if ( fBlockSti
7336 || fBlockMovSS)
7337 {
7338 if (!pVCpu->hm.s.fSingleInstruction)
7339 {
7340 /*
7341 * The pending-debug exceptions field is cleared on all VM-exits except VMX_EXIT_TPR_BELOW_THRESHOLD,
7342 * VMX_EXIT_MTF, VMX_EXIT_APIC_WRITE and VMX_EXIT_VIRTUALIZED_EOI.
7343 * See Intel spec. 27.3.4 "Saving Non-Register State".
7344 */
7345 Assert(!DBGFIsStepping(pVCpu));
7346 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
7347 AssertRCReturn(rc, rc);
7348 if (pMixedCtx->eflags.Bits.u1TF)
7349 {
7350 int rc2 = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
7351 AssertRCReturn(rc2, rc2);
7352 }
7353 }
7354 else if (pMixedCtx->eflags.Bits.u1TF)
7355 {
7356 /*
7357 * We are single-stepping in the hypervisor debugger using EFLAGS.TF. Clear interrupt inhibition as setting the
7358 * BS bit would mean delivering a #DB to the guest upon VM-entry when it shouldn't be.
7359 */
7360 Assert(!(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1 & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG));
7361 fIntrState = 0;
7362 }
7363 }
7364
7365 /*
7366 * There's no need to clear the VM-entry interruption-information field here if we're not injecting anything.
7367 * VT-x clears the valid bit on every VM-exit. See Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
7368 */
7369 int rc3 = hmR0VmxExportGuestIntrState(pVCpu, fIntrState);
7370 AssertRCReturn(rc3, rc3);
7371
7372 Assert(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping));
7373 NOREF(fBlockMovSS); NOREF(fBlockSti);
7374 return rcStrict;
7375}
7376
7377
7378/**
7379 * Sets an invalid-opcode (\#UD) exception as pending-for-injection into the VM.
7380 *
7381 * @param pVCpu The cross context virtual CPU structure.
7382 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7383 * out-of-sync. Make sure to update the required fields
7384 * before using them.
7385 */
7386DECLINLINE(void) hmR0VmxSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7387{
7388 NOREF(pMixedCtx);
7389 uint32_t u32IntInfo = X86_XCPT_UD | VMX_EXIT_INTERRUPTION_INFO_VALID;
7390 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7391}
7392
7393
7394/**
7395 * Injects a double-fault (\#DF) exception into the VM.
7396 *
7397 * @returns Strict VBox status code (i.e. informational status codes too).
7398 * @param pVCpu The cross context virtual CPU structure.
7399 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7400 * out-of-sync. Make sure to update the required fields
7401 * before using them.
7402 * @param fStepping Whether we're running in hmR0VmxRunGuestCodeStep()
7403 * and should return VINF_EM_DBG_STEPPED if the event
7404 * is injected directly (register modified by us, not
7405 * by hardware on VM-entry).
7406 * @param pfIntrState Pointer to the current guest interruptibility-state.
7407 * This interruptibility-state will be updated if
7408 * necessary. This cannot not be NULL.
7409 */
7410DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptDF(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fStepping, uint32_t *pfIntrState)
7411{
7412 NOREF(pMixedCtx);
7413 uint32_t u32IntInfo = X86_XCPT_DF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7414 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7415 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7416 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */, fStepping,
7417 pfIntrState);
7418}
7419
7420
7421/**
7422 * Sets a debug (\#DB) exception as pending-for-injection into the VM.
7423 *
7424 * @param pVCpu The cross context virtual CPU structure.
7425 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7426 * out-of-sync. Make sure to update the required fields
7427 * before using them.
7428 */
7429DECLINLINE(void) hmR0VmxSetPendingXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx)
7430{
7431 NOREF(pMixedCtx);
7432 uint32_t u32IntInfo = X86_XCPT_DB | VMX_EXIT_INTERRUPTION_INFO_VALID;
7433 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7434 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7435}
7436
7437
7438/**
7439 * Sets an overflow (\#OF) exception as pending-for-injection into the VM.
7440 *
7441 * @param pVCpu The cross context virtual CPU structure.
7442 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7443 * out-of-sync. Make sure to update the required fields
7444 * before using them.
7445 * @param cbInstr The value of RIP that is to be pushed on the guest
7446 * stack.
7447 */
7448DECLINLINE(void) hmR0VmxSetPendingXcptOF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
7449{
7450 NOREF(pMixedCtx);
7451 uint32_t u32IntInfo = X86_XCPT_OF | VMX_EXIT_INTERRUPTION_INFO_VALID;
7452 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7453 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7454}
7455
7456
7457/**
7458 * Injects a general-protection (\#GP) fault into the VM.
7459 *
7460 * @returns Strict VBox status code (i.e. informational status codes too).
7461 * @param pVCpu The cross context virtual CPU structure.
7462 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7463 * out-of-sync. Make sure to update the required fields
7464 * before using them.
7465 * @param fErrorCodeValid Whether the error code is valid (depends on the CPU
7466 * mode, i.e. in real-mode it's not valid).
7467 * @param u32ErrorCode The error code associated with the \#GP.
7468 * @param fStepping Whether we're running in
7469 * hmR0VmxRunGuestCodeStep() and should return
7470 * VINF_EM_DBG_STEPPED if the event is injected
7471 * directly (register modified by us, not by
7472 * hardware on VM-entry).
7473 * @param pfIntrState Pointer to the current guest interruptibility-state.
7474 * This interruptibility-state will be updated if
7475 * necessary. This cannot not be NULL.
7476 */
7477DECLINLINE(VBOXSTRICTRC) hmR0VmxInjectXcptGP(PVMCPU pVCpu, PCCPUMCTX pMixedCtx, bool fErrorCodeValid, uint32_t u32ErrorCode,
7478 bool fStepping, uint32_t *pfIntrState)
7479{
7480 NOREF(pMixedCtx);
7481 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7482 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7483 if (fErrorCodeValid)
7484 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7485 return hmR0VmxInjectEventVmcs(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */, fStepping,
7486 pfIntrState);
7487}
7488
7489
7490#if 0 /* unused */
7491/**
7492 * Sets a general-protection (\#GP) exception as pending-for-injection into the
7493 * VM.
7494 *
7495 * @param pVCpu The cross context virtual CPU structure.
7496 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7497 * out-of-sync. Make sure to update the required fields
7498 * before using them.
7499 * @param u32ErrorCode The error code associated with the \#GP.
7500 */
7501DECLINLINE(void) hmR0VmxSetPendingXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t u32ErrorCode)
7502{
7503 NOREF(pMixedCtx);
7504 uint32_t u32IntInfo = X86_XCPT_GP | VMX_EXIT_INTERRUPTION_INFO_VALID;
7505 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7506 u32IntInfo |= VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7507 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, 0 /* cbInstr */, u32ErrorCode, 0 /* GCPtrFaultAddress */);
7508}
7509#endif /* unused */
7510
7511
7512/**
7513 * Sets a software interrupt (INTn) as pending-for-injection into the VM.
7514 *
7515 * @param pVCpu The cross context virtual CPU structure.
7516 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
7517 * out-of-sync. Make sure to update the required fields
7518 * before using them.
7519 * @param uVector The software interrupt vector number.
7520 * @param cbInstr The value of RIP that is to be pushed on the guest
7521 * stack.
7522 */
7523DECLINLINE(void) hmR0VmxSetPendingIntN(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint16_t uVector, uint32_t cbInstr)
7524{
7525 NOREF(pMixedCtx);
7526 uint32_t u32IntInfo = uVector | VMX_EXIT_INTERRUPTION_INFO_VALID;
7527 if ( uVector == X86_XCPT_BP
7528 || uVector == X86_XCPT_OF)
7529 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7530 else
7531 u32IntInfo |= (VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT << VMX_EXIT_INTERRUPTION_INFO_TYPE_SHIFT);
7532 hmR0VmxSetPendingEvent(pVCpu, u32IntInfo, cbInstr, 0 /* u32ErrCode */, 0 /* GCPtrFaultAddress */);
7533}
7534
7535
7536/**
7537 * Pushes a 2-byte value onto the real-mode (in virtual-8086 mode) guest's
7538 * stack.
7539 *
7540 * @returns Strict VBox status code (i.e. informational status codes too).
7541 * @retval VINF_EM_RESET if pushing a value to the stack caused a triple-fault.
7542 * @param pVM The cross context VM structure.
7543 * @param pMixedCtx Pointer to the guest-CPU context.
7544 * @param uValue The value to push to the guest stack.
7545 */
7546DECLINLINE(VBOXSTRICTRC) hmR0VmxRealModeGuestStackPush(PVM pVM, PCPUMCTX pMixedCtx, uint16_t uValue)
7547{
7548 /*
7549 * The stack limit is 0xffff in real-on-virtual 8086 mode. Real-mode with weird stack limits cannot be run in
7550 * virtual 8086 mode in VT-x. See Intel spec. 26.3.1.2 "Checks on Guest Segment Registers".
7551 * See Intel Instruction reference for PUSH and Intel spec. 22.33.1 "Segment Wraparound".
7552 */
7553 if (pMixedCtx->sp == 1)
7554 return VINF_EM_RESET;
7555 pMixedCtx->sp -= sizeof(uint16_t); /* May wrap around which is expected behaviour. */
7556 int rc = PGMPhysSimpleWriteGCPhys(pVM, pMixedCtx->ss.u64Base + pMixedCtx->sp, &uValue, sizeof(uint16_t));
7557 AssertRC(rc);
7558 return rc;
7559}
7560
7561
7562/**
7563 * Injects an event into the guest upon VM-entry by updating the relevant fields
7564 * in the VM-entry area in the VMCS.
7565 *
7566 * @returns Strict VBox status code (i.e. informational status codes too).
7567 * @retval VINF_SUCCESS if the event is successfully injected into the VMCS.
7568 * @retval VINF_EM_RESET if event injection resulted in a triple-fault.
7569 *
7570 * @param pVCpu The cross context virtual CPU structure.
7571 * @param u64IntInfo The VM-entry interruption-information field.
7572 * @param cbInstr The VM-entry instruction length in bytes (for
7573 * software interrupts, exceptions and privileged
7574 * software exceptions).
7575 * @param u32ErrCode The VM-entry exception error code.
7576 * @param GCPtrFaultAddress The page-fault address for \#PF exceptions.
7577 * @param pfIntrState Pointer to the current guest interruptibility-state.
7578 * This interruptibility-state will be updated if
7579 * necessary. This cannot not be NULL.
7580 * @param fStepping Whether we're running in
7581 * hmR0VmxRunGuestCodeStep() and should return
7582 * VINF_EM_DBG_STEPPED if the event is injected
7583 * directly (register modified by us, not by
7584 * hardware on VM-entry).
7585 */
7586static VBOXSTRICTRC hmR0VmxInjectEventVmcs(PVMCPU pVCpu, uint64_t u64IntInfo, uint32_t cbInstr, uint32_t u32ErrCode,
7587 RTGCUINTREG GCPtrFaultAddress, bool fStepping, uint32_t *pfIntrState)
7588{
7589 /* Intel spec. 24.8.3 "VM-Entry Controls for Event Injection" specifies the interruption-information field to be 32-bits. */
7590 AssertMsg(!RT_HI_U32(u64IntInfo), ("%#RX64\n", u64IntInfo));
7591 Assert(pfIntrState);
7592
7593 PCPUMCTX pMixedCtx = &pVCpu->cpum.GstCtx;
7594 uint32_t u32IntInfo = (uint32_t)u64IntInfo;
7595 uint32_t const uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(u32IntInfo);
7596 uint32_t const uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo);
7597
7598#ifdef VBOX_STRICT
7599 /*
7600 * Validate the error-code-valid bit for hardware exceptions.
7601 * No error codes for exceptions in real-mode.
7602 *
7603 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7604 */
7605 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7606 && !CPUMIsGuestInRealModeEx(pMixedCtx))
7607 {
7608 switch (uVector)
7609 {
7610 case X86_XCPT_PF:
7611 case X86_XCPT_DF:
7612 case X86_XCPT_TS:
7613 case X86_XCPT_NP:
7614 case X86_XCPT_SS:
7615 case X86_XCPT_GP:
7616 case X86_XCPT_AC:
7617 AssertMsg(VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo),
7618 ("Error-code-valid bit not set for exception that has an error code uVector=%#x\n", uVector));
7619 RT_FALL_THRU();
7620 default:
7621 break;
7622 }
7623 }
7624#endif
7625
7626 /* Cannot inject an NMI when block-by-MOV SS is in effect. */
7627 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7628 || !(*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS));
7629
7630 STAM_COUNTER_INC(&pVCpu->hm.s.paStatInjectedIrqsR0[uVector & MASK_INJECT_IRQ_STAT]);
7631
7632 /*
7633 * Hardware interrupts & exceptions cannot be delivered through the software interrupt
7634 * redirection bitmap to the real mode task in virtual-8086 mode. We must jump to the
7635 * interrupt handler in the (real-mode) guest.
7636 *
7637 * See Intel spec. 20.3 "Interrupt and Exception handling in Virtual-8086 Mode".
7638 * See Intel spec. 20.1.4 "Interrupt and Exception Handling" for real-mode interrupt handling.
7639 */
7640 if (CPUMIsGuestInRealModeEx(pMixedCtx)) /* CR0.PE bit changes are always intercepted, so it's up to date. */
7641 {
7642 if (pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest)
7643 {
7644 /*
7645 * For unrestricted execution enabled CPUs running real-mode guests, we must not
7646 * set the deliver-error-code bit.
7647 *
7648 * See Intel spec. 26.2.1.3 "VM-Entry Control Fields".
7649 */
7650 u32IntInfo &= ~VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_VALID;
7651 }
7652 else
7653 {
7654 PVM pVM = pVCpu->CTX_SUFF(pVM);
7655 Assert(PDMVmmDevHeapIsEnabled(pVM));
7656 Assert(pVM->hm.s.vmx.pRealModeTSS);
7657
7658 /* We require RIP, RSP, RFLAGS, CS, IDTR, import them. */
7659 int rc2 = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
7660 | CPUMCTX_EXTRN_TABLE_MASK
7661 | CPUMCTX_EXTRN_RIP
7662 | CPUMCTX_EXTRN_RSP
7663 | CPUMCTX_EXTRN_RFLAGS);
7664 AssertRCReturn(rc2, rc2);
7665
7666 /* Check if the interrupt handler is present in the IVT (real-mode IDT). IDT limit is (4N - 1). */
7667 size_t const cbIdtEntry = sizeof(X86IDTR16);
7668 if (uVector * cbIdtEntry + (cbIdtEntry - 1) > pMixedCtx->idtr.cbIdt)
7669 {
7670 /* If we are trying to inject a #DF with no valid IDT entry, return a triple-fault. */
7671 if (uVector == X86_XCPT_DF)
7672 return VINF_EM_RESET;
7673
7674 /* If we're injecting a #GP with no valid IDT entry, inject a double-fault. */
7675 if (uVector == X86_XCPT_GP)
7676 return hmR0VmxInjectXcptDF(pVCpu, pMixedCtx, fStepping, pfIntrState);
7677
7678 /*
7679 * If we're injecting an event with no valid IDT entry, inject a #GP.
7680 * No error codes for exceptions in real-mode.
7681 *
7682 * See Intel spec. 20.1.4 "Interrupt and Exception Handling"
7683 */
7684 return hmR0VmxInjectXcptGP(pVCpu, pMixedCtx, false /* fErrCodeValid */, 0 /* u32ErrCode */, fStepping,
7685 pfIntrState);
7686 }
7687
7688 /* Software exceptions (#BP and #OF exceptions thrown as a result of INT3 or INTO) */
7689 uint16_t uGuestIp = pMixedCtx->ip;
7690 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT)
7691 {
7692 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF);
7693 /* #BP and #OF are both benign traps, we need to resume the next instruction. */
7694 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7695 }
7696 else if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT)
7697 uGuestIp = pMixedCtx->ip + (uint16_t)cbInstr;
7698
7699 /* Get the code segment selector and offset from the IDT entry for the interrupt handler. */
7700 X86IDTR16 IdtEntry;
7701 RTGCPHYS GCPhysIdtEntry = (RTGCPHYS)pMixedCtx->idtr.pIdt + uVector * cbIdtEntry;
7702 rc2 = PGMPhysSimpleReadGCPhys(pVM, &IdtEntry, GCPhysIdtEntry, cbIdtEntry);
7703 AssertRCReturn(rc2, rc2);
7704
7705 /* Construct the stack frame for the interrupt/exception handler. */
7706 VBOXSTRICTRC rcStrict;
7707 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->eflags.u32);
7708 if (rcStrict == VINF_SUCCESS)
7709 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, pMixedCtx->cs.Sel);
7710 if (rcStrict == VINF_SUCCESS)
7711 rcStrict = hmR0VmxRealModeGuestStackPush(pVM, pMixedCtx, uGuestIp);
7712
7713 /* Clear the required eflag bits and jump to the interrupt/exception handler. */
7714 if (rcStrict == VINF_SUCCESS)
7715 {
7716 pMixedCtx->eflags.u32 &= ~(X86_EFL_IF | X86_EFL_TF | X86_EFL_RF | X86_EFL_AC);
7717 pMixedCtx->rip = IdtEntry.offSel;
7718 pMixedCtx->cs.Sel = IdtEntry.uSel;
7719 pMixedCtx->cs.ValidSel = IdtEntry.uSel;
7720 pMixedCtx->cs.u64Base = IdtEntry.uSel << cbIdtEntry;
7721 if ( uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7722 && uVector == X86_XCPT_PF)
7723 pMixedCtx->cr2 = GCPtrFaultAddress;
7724
7725 /* If any other guest-state bits are changed here, make sure to update
7726 hmR0VmxPreRunGuestCommitted() when thread-context hooks are used. */
7727 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CS
7728 | HM_CHANGED_GUEST_CR2
7729 | HM_CHANGED_GUEST_RIP
7730 | HM_CHANGED_GUEST_RFLAGS
7731 | HM_CHANGED_GUEST_RSP);
7732
7733 /* We're clearing interrupts, which means no block-by-STI interrupt-inhibition. */
7734 if (*pfIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
7735 {
7736 Assert( uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI
7737 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
7738 Log4Func(("Clearing inhibition due to STI\n"));
7739 *pfIntrState &= ~VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI;
7740 }
7741 Log4(("Injecting real-mode: u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x Eflags=%#x CS:EIP=%04x:%04x\n",
7742 u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->eflags.u, pMixedCtx->cs.Sel, pMixedCtx->eip));
7743
7744 /* The event has been truly dispatched. Mark it as no longer pending so we don't attempt to 'undo'
7745 it, if we are returning to ring-3 before executing guest code. */
7746 pVCpu->hm.s.Event.fPending = false;
7747
7748 /* Make hmR0VmxPreRunGuest() return if we're stepping since we've changed cs:rip. */
7749 if (fStepping)
7750 rcStrict = VINF_EM_DBG_STEPPED;
7751 }
7752 AssertMsg(rcStrict == VINF_SUCCESS || rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
7753 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
7754 return rcStrict;
7755 }
7756 }
7757
7758 /* Validate. */
7759 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(u32IntInfo)); /* Bit 31 (Valid bit) must be set by caller. */
7760 Assert(!VMX_EXIT_INTERRUPTION_INFO_NMI_UNBLOCK_IRET(u32IntInfo)); /* Bit 12 MBZ. */
7761 Assert(!(u32IntInfo & 0x7ffff000)); /* Bits 30:12 MBZ. */
7762
7763 /* Inject. */
7764 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, u32IntInfo);
7765 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(u32IntInfo))
7766 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE, u32ErrCode);
7767 rc |= VMXWriteVmcs32(VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH, cbInstr);
7768 AssertRCReturn(rc, rc);
7769
7770 /* Update CR2. */
7771 if ( VMX_EXIT_INTERRUPTION_INFO_TYPE(u32IntInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT
7772 && uVector == X86_XCPT_PF)
7773 pMixedCtx->cr2 = GCPtrFaultAddress;
7774
7775 Log4(("Injecting u32IntInfo=%#x u32ErrCode=%#x cbInstr=%#x CR2=%#RX64\n", u32IntInfo, u32ErrCode, cbInstr, pMixedCtx->cr2));
7776
7777 return VINF_SUCCESS;
7778}
7779
7780
7781/**
7782 * Clears the interrupt-window exiting control in the VMCS and if necessary
7783 * clears the current event in the VMCS as well.
7784 *
7785 * @returns VBox status code.
7786 * @param pVCpu The cross context virtual CPU structure.
7787 *
7788 * @remarks Use this function only to clear events that have not yet been
7789 * delivered to the guest but are injected in the VMCS!
7790 * @remarks No-long-jump zone!!!
7791 */
7792static void hmR0VmxClearIntNmiWindowsVmcs(PVMCPU pVCpu)
7793{
7794 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_INT_WINDOW_EXIT)
7795 {
7796 hmR0VmxClearIntWindowExitVmcs(pVCpu);
7797 Log4Func(("Cleared interrupt widow\n"));
7798 }
7799
7800 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)
7801 {
7802 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
7803 Log4Func(("Cleared interrupt widow\n"));
7804 }
7805}
7806
7807
7808/**
7809 * Enters the VT-x session.
7810 *
7811 * @returns VBox status code.
7812 * @param pVCpu The cross context virtual CPU structure.
7813 * @param pHostCpu Pointer to the global CPU info struct.
7814 */
7815VMMR0DECL(int) VMXR0Enter(PVMCPU pVCpu, PHMGLOBALCPUINFO pHostCpu)
7816{
7817 AssertPtr(pVCpu);
7818 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fSupported);
7819 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7820 RT_NOREF(pHostCpu);
7821
7822 LogFlowFunc(("pVCpu=%p\n", pVCpu));
7823 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7824 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7825
7826#ifdef VBOX_STRICT
7827 /* At least verify VMX is enabled, since we can't check if we're in VMX root mode without #GP'ing. */
7828 RTCCUINTREG uHostCR4 = ASMGetCR4();
7829 if (!(uHostCR4 & X86_CR4_VMXE))
7830 {
7831 LogRelFunc(("X86_CR4_VMXE bit in CR4 is not set!\n"));
7832 return VERR_VMX_X86_CR4_VMXE_CLEARED;
7833 }
7834#endif
7835
7836 /*
7837 * Load the VCPU's VMCS as the current (and active) one.
7838 */
7839 Assert(pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR);
7840 int rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7841 if (RT_FAILURE(rc))
7842 return rc;
7843
7844 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7845 pVCpu->hm.s.fLeaveDone = false;
7846 Log4Func(("Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7847
7848 return VINF_SUCCESS;
7849}
7850
7851
7852/**
7853 * The thread-context callback (only on platforms which support it).
7854 *
7855 * @param enmEvent The thread-context event.
7856 * @param pVCpu The cross context virtual CPU structure.
7857 * @param fGlobalInit Whether global VT-x/AMD-V init. was used.
7858 * @thread EMT(pVCpu)
7859 */
7860VMMR0DECL(void) VMXR0ThreadCtxCallback(RTTHREADCTXEVENT enmEvent, PVMCPU pVCpu, bool fGlobalInit)
7861{
7862 NOREF(fGlobalInit);
7863
7864 switch (enmEvent)
7865 {
7866 case RTTHREADCTXEVENT_OUT:
7867 {
7868 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7869 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7870 VMCPU_ASSERT_EMT(pVCpu);
7871
7872 /* No longjmps (logger flushes, locks) in this fragile context. */
7873 VMMRZCallRing3Disable(pVCpu);
7874 Log4Func(("Preempting: HostCpuId=%u\n", RTMpCpuId()));
7875
7876 /*
7877 * Restore host-state (FPU, debug etc.)
7878 */
7879 if (!pVCpu->hm.s.fLeaveDone)
7880 {
7881 /*
7882 * Do -not- import the guest-state here as we might already be in the middle of importing
7883 * it, esp. bad if we're holding the PGM lock, see comment in hmR0VmxImportGuestState().
7884 */
7885 hmR0VmxLeave(pVCpu, false /* fImportState */);
7886 pVCpu->hm.s.fLeaveDone = true;
7887 }
7888
7889 /* Leave HM context, takes care of local init (term). */
7890 int rc = HMR0LeaveCpu(pVCpu);
7891 AssertRC(rc); NOREF(rc);
7892
7893 /* Restore longjmp state. */
7894 VMMRZCallRing3Enable(pVCpu);
7895 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreempt);
7896 break;
7897 }
7898
7899 case RTTHREADCTXEVENT_IN:
7900 {
7901 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7902 Assert(VMMR0ThreadCtxHookIsEnabled(pVCpu));
7903 VMCPU_ASSERT_EMT(pVCpu);
7904
7905 /* No longjmps here, as we don't want to trigger preemption (& its hook) while resuming. */
7906 VMMRZCallRing3Disable(pVCpu);
7907 Log4Func(("Resumed: HostCpuId=%u\n", RTMpCpuId()));
7908
7909 /* Initialize the bare minimum state required for HM. This takes care of
7910 initializing VT-x if necessary (onlined CPUs, local init etc.) */
7911 int rc = hmR0EnterCpu(pVCpu);
7912 AssertRC(rc);
7913 Assert((pVCpu->hm.s.fCtxChanged & (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
7914 == (HM_CHANGED_HOST_CONTEXT | HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE));
7915
7916 /* Load the active VMCS as the current one. */
7917 if (pVCpu->hm.s.vmx.uVmcsState & HMVMX_VMCS_STATE_CLEAR)
7918 {
7919 rc = VMXActivateVmcs(pVCpu->hm.s.vmx.HCPhysVmcs);
7920 AssertRC(rc); NOREF(rc);
7921 pVCpu->hm.s.vmx.uVmcsState = HMVMX_VMCS_STATE_ACTIVE;
7922 Log4Func(("Resumed: Activated Vmcs. HostCpuId=%u\n", RTMpCpuId()));
7923 }
7924 pVCpu->hm.s.fLeaveDone = false;
7925
7926 /* Restore longjmp state. */
7927 VMMRZCallRing3Enable(pVCpu);
7928 break;
7929 }
7930
7931 default:
7932 break;
7933 }
7934}
7935
7936
7937/**
7938 * Exports the host state into the VMCS host-state area.
7939 * Sets up the VM-exit MSR-load area.
7940 *
7941 * The CPU state will be loaded from these fields on every successful VM-exit.
7942 *
7943 * @returns VBox status code.
7944 * @param pVCpu The cross context virtual CPU structure.
7945 *
7946 * @remarks No-long-jump zone!!!
7947 */
7948static int hmR0VmxExportHostState(PVMCPU pVCpu)
7949{
7950 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7951
7952 int rc = VINF_SUCCESS;
7953 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
7954 {
7955 rc = hmR0VmxExportHostControlRegs();
7956 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7957
7958 rc = hmR0VmxExportHostSegmentRegs(pVCpu);
7959 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7960
7961 rc = hmR0VmxExportHostMsrs(pVCpu);
7962 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
7963
7964 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_HOST_CONTEXT;
7965 }
7966 return rc;
7967}
7968
7969
7970/**
7971 * Saves the host state in the VMCS host-state.
7972 *
7973 * @returns VBox status code.
7974 * @param pVCpu The cross context virtual CPU structure.
7975 *
7976 * @remarks No-long-jump zone!!!
7977 */
7978VMMR0DECL(int) VMXR0ExportHostState(PVMCPU pVCpu)
7979{
7980 AssertPtr(pVCpu);
7981 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
7982
7983 /*
7984 * Export the host state here while entering HM context.
7985 * When thread-context hooks are used, we might get preempted and have to re-save the host
7986 * state but most of the time we won't be, so do it here before we disable interrupts.
7987 */
7988 return hmR0VmxExportHostState(pVCpu);
7989}
7990
7991
7992/**
7993 * Exports the guest state into the VMCS guest-state area.
7994 *
7995 * The will typically be done before VM-entry when the guest-CPU state and the
7996 * VMCS state may potentially be out of sync.
7997 *
7998 * Sets up the VM-entry MSR-load and VM-exit MSR-store areas. Sets up the
7999 * VM-entry controls.
8000 * Sets up the appropriate VMX non-root function to execute guest code based on
8001 * the guest CPU mode.
8002 *
8003 * @returns VBox strict status code.
8004 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8005 * without unrestricted guest access and the VMMDev is not presently
8006 * mapped (e.g. EFI32).
8007 *
8008 * @param pVCpu The cross context virtual CPU structure.
8009 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8010 * out-of-sync. Make sure to update the required fields
8011 * before using them.
8012 *
8013 * @remarks No-long-jump zone!!!
8014 */
8015static VBOXSTRICTRC hmR0VmxExportGuestState(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8016{
8017 AssertPtr(pVCpu);
8018 AssertPtr(pMixedCtx);
8019 HMVMX_ASSERT_PREEMPT_SAFE();
8020
8021 LogFlowFunc(("pVCpu=%p\n", pVCpu));
8022
8023 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExportGuestState, x);
8024
8025 /* Determine real-on-v86 mode. */
8026 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = false;
8027 if ( !pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest
8028 && CPUMIsGuestInRealModeEx(pMixedCtx))
8029 {
8030 pVCpu->hm.s.vmx.RealMode.fRealOnV86Active = true;
8031 }
8032
8033 /*
8034 * Any ordering dependency among the sub-functions below must be explicitly stated using comments.
8035 * Ideally, assert that the cross-dependent bits are up-to-date at the point of using it.
8036 */
8037 int rc = hmR0VmxSelectVMRunHandler(pVCpu, pMixedCtx);
8038 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8039
8040 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-entry control updates. */
8041 rc = hmR0VmxExportGuestEntryCtls(pVCpu, pMixedCtx);
8042 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8043
8044 /* This needs to be done after hmR0VmxSelectVMRunHandler() as changing pfnStartVM may require VM-exit control updates. */
8045 rc = hmR0VmxExportGuestExitCtls(pVCpu, pMixedCtx);
8046 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8047
8048 rc = hmR0VmxExportGuestCR0(pVCpu, pMixedCtx);
8049 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8050
8051 VBOXSTRICTRC rcStrict = hmR0VmxExportGuestCR3AndCR4(pVCpu, pMixedCtx);
8052 if (rcStrict == VINF_SUCCESS)
8053 { /* likely */ }
8054 else
8055 {
8056 Assert(rcStrict == VINF_EM_RESCHEDULE_REM || RT_FAILURE_NP(rcStrict));
8057 return rcStrict;
8058 }
8059
8060 rc = hmR0VmxExportGuestSegmentRegs(pVCpu, pMixedCtx);
8061 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8062
8063 /* This needs to be done after hmR0VmxExportGuestEntryCtls() and hmR0VmxExportGuestExitCtls() as it
8064 may alter controls if we determine we don't have to swap EFER after all. */
8065 rc = hmR0VmxExportGuestMsrs(pVCpu, pMixedCtx);
8066 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8067
8068 rc = hmR0VmxExportGuestApicTpr(pVCpu);
8069 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8070
8071 /* This needs to be done after hmR0VmxExportGuestCR0() as it may alter intercepted exceptions. */
8072 rc = hmR0VmxExportGuestXcptIntercepts(pVCpu);
8073 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8074
8075 /* Exporting RFLAGS here is fine, even though RFLAGS.TF might depend on guest debug state which is
8076 not exported here. It is re-evaluated and updated if necessary in hmR0VmxExportSharedState(). */
8077 rc = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8078 rc |= hmR0VmxExportGuestRsp(pVCpu, pMixedCtx);
8079 rc |= hmR0VmxExportGuestRflags(pVCpu, pMixedCtx);
8080 AssertLogRelMsgRCReturn(rc, ("rc=%Rrc\n", rc), rc);
8081
8082 /* Clear any bits that may be set but exported unconditionally or unused/reserved bits. */
8083 ASMAtomicUoAndU64(&pVCpu->hm.s.fCtxChanged, ~( (HM_CHANGED_GUEST_GPRS_MASK & ~HM_CHANGED_GUEST_RSP)
8084 | HM_CHANGED_GUEST_CR2
8085 | (HM_CHANGED_GUEST_DR_MASK & ~HM_CHANGED_GUEST_DR7)
8086 | HM_CHANGED_GUEST_X87
8087 | HM_CHANGED_GUEST_SSE_AVX
8088 | HM_CHANGED_GUEST_OTHER_XSAVE
8089 | HM_CHANGED_GUEST_XCRx
8090 | HM_CHANGED_GUEST_KERNEL_GS_BASE /* Part of lazy or auto load-store MSRs. */
8091 | HM_CHANGED_GUEST_SYSCALL_MSRS /* Part of lazy or auto load-store MSRs. */
8092 | HM_CHANGED_GUEST_TSC_AUX
8093 | HM_CHANGED_GUEST_OTHER_MSRS
8094 | HM_CHANGED_GUEST_HWVIRT
8095 | (HM_CHANGED_KEEPER_STATE_MASK & ~HM_CHANGED_VMX_MASK)));
8096
8097 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExportGuestState, x);
8098 return rc;
8099}
8100
8101
8102/**
8103 * Exports the state shared between the host and guest into the VMCS.
8104 *
8105 * @param pVCpu The cross context virtual CPU structure.
8106 * @param pCtx Pointer to the guest-CPU context.
8107 *
8108 * @remarks No-long-jump zone!!!
8109 */
8110static void hmR0VmxExportSharedState(PVMCPU pVCpu, PCPUMCTX pCtx)
8111{
8112 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8113 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8114
8115 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_DR_MASK)
8116 {
8117 int rc = hmR0VmxExportSharedDebugState(pVCpu, pCtx);
8118 AssertRC(rc);
8119 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_GUEST_DR_MASK;
8120
8121 /* Loading shared debug bits might have changed eflags.TF bit for debugging purposes. */
8122 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_GUEST_RFLAGS)
8123 {
8124 rc = hmR0VmxExportGuestRflags(pVCpu, pCtx);
8125 AssertRC(rc);
8126 }
8127 }
8128
8129 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_GUEST_LAZY_MSRS)
8130 {
8131 hmR0VmxLazyLoadGuestMsrs(pVCpu, pCtx);
8132 pVCpu->hm.s.fCtxChanged &= ~HM_CHANGED_VMX_GUEST_LAZY_MSRS;
8133 }
8134
8135 AssertMsg(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE),
8136 ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8137}
8138
8139
8140/**
8141 * Worker for loading the guest-state bits in the inner VT-x execution loop.
8142 *
8143 * @returns Strict VBox status code (i.e. informational status codes too).
8144 * @retval VINF_EM_RESCHEDULE_REM if we try to emulate non-paged guest code
8145 * without unrestricted guest access and the VMMDev is not presently
8146 * mapped (e.g. EFI32).
8147 *
8148 * @param pVCpu The cross context virtual CPU structure.
8149 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8150 * out-of-sync. Make sure to update the required fields
8151 * before using them.
8152 *
8153 * @remarks No-long-jump zone!!!
8154 */
8155static VBOXSTRICTRC hmR0VmxExportGuestStateOptimal(PVMCPU pVCpu, PCCPUMCTX pMixedCtx)
8156{
8157 HMVMX_ASSERT_PREEMPT_SAFE();
8158 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8159 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8160
8161#ifdef HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE
8162 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
8163#endif
8164
8165 /*
8166 * For many exits it's only RIP that changes and hence try to export it first
8167 * without going through a lot of change flag checks.
8168 */
8169 VBOXSTRICTRC rcStrict;
8170 uint64_t fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8171 if ((fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)) == HM_CHANGED_GUEST_RIP)
8172 {
8173 rcStrict = hmR0VmxExportGuestRip(pVCpu, pMixedCtx);
8174 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8175 { /* likely */}
8176 else
8177 AssertMsgFailedReturn(("hmR0VmxExportGuestRip failed! rc=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)), rcStrict);
8178 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportMinimal);
8179 }
8180 else if (fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE))
8181 {
8182 rcStrict = hmR0VmxExportGuestState(pVCpu, pMixedCtx);
8183 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8184 { /* likely */}
8185 else
8186 {
8187 AssertMsg(rcStrict == VINF_EM_RESCHEDULE_REM, ("hmR0VmxExportGuestState failed! rc=%Rrc\n",
8188 VBOXSTRICTRC_VAL(rcStrict)));
8189 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8190 return rcStrict;
8191 }
8192 STAM_COUNTER_INC(&pVCpu->hm.s.StatExportFull);
8193 }
8194 else
8195 rcStrict = VINF_SUCCESS;
8196
8197#ifdef VBOX_STRICT
8198 /* All the guest state bits should be loaded except maybe the host context and/or the shared host/guest bits. */
8199 fCtxChanged = ASMAtomicUoReadU64(&pVCpu->hm.s.fCtxChanged);
8200 AssertMsg(!(fCtxChanged & (HM_CHANGED_ALL_GUEST & ~HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)),
8201 ("fCtxChanged=%#RX64\n", fCtxChanged));
8202#endif
8203 return rcStrict;
8204}
8205
8206
8207/**
8208 * Does the preparations before executing guest code in VT-x.
8209 *
8210 * This may cause longjmps to ring-3 and may even result in rescheduling to the
8211 * recompiler/IEM. We must be cautious what we do here regarding committing
8212 * guest-state information into the VMCS assuming we assuredly execute the
8213 * guest in VT-x mode.
8214 *
8215 * If we fall back to the recompiler/IEM after updating the VMCS and clearing
8216 * the common-state (TRPM/forceflags), we must undo those changes so that the
8217 * recompiler/IEM can (and should) use them when it resumes guest execution.
8218 * Otherwise such operations must be done when we can no longer exit to ring-3.
8219 *
8220 * @returns Strict VBox status code (i.e. informational status codes too).
8221 * @retval VINF_SUCCESS if we can proceed with running the guest, interrupts
8222 * have been disabled.
8223 * @retval VINF_EM_RESET if a triple-fault occurs while injecting a
8224 * double-fault into the guest.
8225 * @retval VINF_EM_DBG_STEPPED if @a fStepping is true and an event was
8226 * dispatched directly.
8227 * @retval VINF_* scheduling changes, we have to go back to ring-3.
8228 *
8229 * @param pVCpu The cross context virtual CPU structure.
8230 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8231 * out-of-sync. Make sure to update the required fields
8232 * before using them.
8233 * @param pVmxTransient Pointer to the VMX transient structure.
8234 * @param fStepping Set if called from hmR0VmxRunGuestCodeStep(). Makes
8235 * us ignore some of the reasons for returning to
8236 * ring-3, and return VINF_EM_DBG_STEPPED if event
8237 * dispatching took place.
8238 */
8239static VBOXSTRICTRC hmR0VmxPreRunGuest(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, bool fStepping)
8240{
8241 Assert(VMMRZCallRing3IsEnabled(pVCpu));
8242
8243#ifdef VBOX_WITH_2X_4GB_ADDR_SPACE_IN_R0
8244 PGMRZDynMapFlushAutoSet(pVCpu);
8245#endif
8246
8247 /* Check force flag actions that might require us to go back to ring-3. */
8248 VBOXSTRICTRC rcStrict = hmR0VmxCheckForceFlags(pVCpu, pMixedCtx, fStepping);
8249 if (rcStrict == VINF_SUCCESS)
8250 { /* FFs doesn't get set all the time. */ }
8251 else
8252 return rcStrict;
8253
8254 /*
8255 * Setup the virtualized-APIC accesses.
8256 *
8257 * Note! This can cause a longjumps to R3 due to the acquisition of the PGM lock
8258 * in both PGMHandlerPhysicalReset() and IOMMMIOMapMMIOHCPage(), see @bugref{8721}.
8259 *
8260 * This is the reason we do it here and not in hmR0VmxExportGuestState().
8261 */
8262 PVM pVM = pVCpu->CTX_SUFF(pVM);
8263 if ( !pVCpu->hm.s.vmx.u64MsrApicBase
8264 && (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_VIRT_APIC)
8265 && PDMHasApic(pVM))
8266 {
8267 uint64_t const u64MsrApicBase = APICGetBaseMsrNoCheck(pVCpu);
8268 Assert(u64MsrApicBase);
8269 Assert(pVM->hm.s.vmx.HCPhysApicAccess);
8270
8271 RTGCPHYS const GCPhysApicBase = u64MsrApicBase & PAGE_BASE_GC_MASK;
8272
8273 /* Unalias any existing mapping. */
8274 int rc = PGMHandlerPhysicalReset(pVM, GCPhysApicBase);
8275 AssertRCReturn(rc, rc);
8276
8277 /* Map the HC APIC-access page in place of the MMIO page, also updates the shadow page tables if necessary. */
8278 Log4Func(("Mapped HC APIC-access page at %#RGp\n", GCPhysApicBase));
8279 rc = IOMMMIOMapMMIOHCPage(pVM, pVCpu, GCPhysApicBase, pVM->hm.s.vmx.HCPhysApicAccess, X86_PTE_RW | X86_PTE_P);
8280 AssertRCReturn(rc, rc);
8281
8282 /* Update the per-VCPU cache of the APIC base MSR. */
8283 pVCpu->hm.s.vmx.u64MsrApicBase = u64MsrApicBase;
8284 }
8285
8286 if (TRPMHasTrap(pVCpu))
8287 hmR0VmxTrpmTrapToPendingEvent(pVCpu);
8288 uint32_t fIntrState = hmR0VmxEvaluatePendingEvent(pVCpu, pMixedCtx);
8289
8290 /*
8291 * Event injection may take locks (currently the PGM lock for real-on-v86 case) and thus
8292 * needs to be done with longjmps or interrupts + preemption enabled. Event injection might
8293 * also result in triple-faulting the VM.
8294 */
8295 rcStrict = hmR0VmxInjectPendingEvent(pVCpu, pMixedCtx, fIntrState, fStepping);
8296 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8297 { /* likely */ }
8298 else
8299 {
8300 AssertMsg(rcStrict == VINF_EM_RESET || (rcStrict == VINF_EM_DBG_STEPPED && fStepping),
8301 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
8302 return rcStrict;
8303 }
8304
8305 /*
8306 * A longjump might result in importing CR3 even for VM-exits that don't necessarily
8307 * import CR3 themselves. We will need to update them here as even as late as the above
8308 * hmR0VmxInjectPendingEvent() call may lazily import guest-CPU state on demand causing
8309 * the below force flags to be set.
8310 */
8311 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3))
8312 {
8313 Assert(!(ASMAtomicUoReadU64(&pMixedCtx->fExtrn) & CPUMCTX_EXTRN_CR3));
8314 int rc2 = PGMUpdateCR3(pVCpu, CPUMGetGuestCR3(pVCpu));
8315 AssertMsgReturn(rc2 == VINF_SUCCESS || rc2 == VINF_PGM_SYNC_CR3,
8316 ("%Rrc\n", rc2), RT_FAILURE_NP(rc2) ? rc2 : VERR_IPE_UNEXPECTED_INFO_STATUS);
8317 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8318 }
8319 if (VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES))
8320 {
8321 PGMGstUpdatePaePdpes(pVCpu, &pVCpu->hm.s.aPdpes[0]);
8322 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8323 }
8324
8325 /*
8326 * No longjmps to ring-3 from this point on!!!
8327 * Asserts() will still longjmp to ring-3 (but won't return), which is intentional, better than a kernel panic.
8328 * This also disables flushing of the R0-logger instance (if any).
8329 */
8330 VMMRZCallRing3Disable(pVCpu);
8331
8332 /*
8333 * Export the guest state bits.
8334 *
8335 * We cannot perform longjmps while loading the guest state because we do not preserve the
8336 * host/guest state (although the VMCS will be preserved) across longjmps which can cause
8337 * CPU migration.
8338 *
8339 * If we are injecting events to a real-on-v86 mode guest, we will have to update
8340 * RIP and some segment registers, i.e. hmR0VmxInjectPendingEvent()->hmR0VmxInjectEventVmcs().
8341 * Hence, loading of the guest state needs to be done -after- injection of events.
8342 */
8343 rcStrict = hmR0VmxExportGuestStateOptimal(pVCpu, pMixedCtx);
8344 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
8345 { /* likely */ }
8346 else
8347 {
8348 VMMRZCallRing3Enable(pVCpu);
8349 return rcStrict;
8350 }
8351
8352 /*
8353 * We disable interrupts so that we don't miss any interrupts that would flag preemption
8354 * (IPI/timers etc.) when thread-context hooks aren't used and we've been running with
8355 * preemption disabled for a while. Since this is purly to aid the
8356 * RTThreadPreemptIsPending() code, it doesn't matter that it may temporarily reenable and
8357 * disable interrupt on NT.
8358 *
8359 * We need to check for force-flags that could've possible been altered since we last
8360 * checked them (e.g. by PDMGetInterrupt() leaving the PDM critical section,
8361 * see @bugref{6398}).
8362 *
8363 * We also check a couple of other force-flags as a last opportunity to get the EMT back
8364 * to ring-3 before executing guest code.
8365 */
8366 pVmxTransient->fEFlags = ASMIntDisableFlags();
8367
8368 if ( ( !VM_FF_IS_PENDING(pVM, VM_FF_EMT_RENDEZVOUS | VM_FF_TM_VIRTUAL_SYNC)
8369 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK))
8370 || ( fStepping /* Optimized for the non-stepping case, so a bit of unnecessary work when stepping. */
8371 && !VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_TO_R3_MASK & ~(VMCPU_FF_TIMER | VMCPU_FF_PDM_CRITSECT))) )
8372 {
8373 if (!RTThreadPreemptIsPending(NIL_RTTHREAD))
8374 {
8375 pVCpu->hm.s.Event.fPending = false;
8376
8377 /*
8378 * We've injected any pending events. This is really the point of no return (to ring-3).
8379 *
8380 * Note! The caller expects to continue with interrupts & longjmps disabled on successful
8381 * returns from this function, so don't enable them here.
8382 */
8383 return VINF_SUCCESS;
8384 }
8385
8386 STAM_COUNTER_INC(&pVCpu->hm.s.StatPendingHostIrq);
8387 rcStrict = VINF_EM_RAW_INTERRUPT;
8388 }
8389 else
8390 {
8391 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHmToR3FF);
8392 rcStrict = VINF_EM_RAW_TO_R3;
8393 }
8394
8395 ASMSetFlags(pVmxTransient->fEFlags);
8396 VMMRZCallRing3Enable(pVCpu);
8397
8398 return rcStrict;
8399}
8400
8401
8402/**
8403 * Prepares to run guest code in VT-x and we've committed to doing so. This
8404 * means there is no backing out to ring-3 or anywhere else at this
8405 * point.
8406 *
8407 * @param pVCpu The cross context virtual CPU structure.
8408 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
8409 * out-of-sync. Make sure to update the required fields
8410 * before using them.
8411 * @param pVmxTransient Pointer to the VMX transient structure.
8412 *
8413 * @remarks Called with preemption disabled.
8414 * @remarks No-long-jump zone!!!
8415 */
8416static void hmR0VmxPreRunGuestCommitted(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
8417{
8418 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8419 Assert(VMMR0IsLogFlushDisabled(pVCpu));
8420 Assert(!RTThreadPreemptIsEnabled(NIL_RTTHREAD));
8421
8422 /*
8423 * Indicate start of guest execution and where poking EMT out of guest-context is recognized.
8424 */
8425 VMCPU_ASSERT_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8426 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_EXEC);
8427
8428 PVM pVM = pVCpu->CTX_SUFF(pVM);
8429 if (!CPUMIsGuestFPUStateActive(pVCpu))
8430 {
8431 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8432 if (CPUMR0LoadGuestFPU(pVM, pVCpu) == VINF_CPUM_HOST_CR0_MODIFIED)
8433 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_HOST_CONTEXT;
8434 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatLoadGuestFpuState, x);
8435 STAM_COUNTER_INC(&pVCpu->hm.s.StatLoadGuestFpu);
8436 }
8437
8438 /*
8439 * Lazy-update of the host MSRs values in the auto-load/store MSR area.
8440 */
8441 if ( !pVCpu->hm.s.vmx.fUpdatedHostMsrs
8442 && pVCpu->hm.s.vmx.cMsrs > 0)
8443 hmR0VmxUpdateAutoLoadStoreHostMsrs(pVCpu);
8444
8445 /*
8446 * Re-save the host state bits as we may've been preempted (only happens when
8447 * thread-context hooks are used or when hmR0VmxSetupVMRunHandler() changes pfnStartVM).
8448 * Note that the 64-on-32 switcher saves the (64-bit) host state into the VMCS and
8449 * if we change the switcher back to 32-bit, we *must* save the 32-bit host state here.
8450 * See @bugref{8432}.
8451 */
8452 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT)
8453 {
8454 int rc = hmR0VmxExportHostState(pVCpu);
8455 AssertRC(rc);
8456 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchPreemptExportHostState);
8457 }
8458 Assert(!(pVCpu->hm.s.fCtxChanged & HM_CHANGED_HOST_CONTEXT));
8459
8460 /*
8461 * Export the state shared between host and guest (FPU, debug, lazy MSRs).
8462 */
8463 if (pVCpu->hm.s.fCtxChanged & HM_CHANGED_VMX_HOST_GUEST_SHARED_STATE)
8464 hmR0VmxExportSharedState(pVCpu, pMixedCtx);
8465 AssertMsg(!pVCpu->hm.s.fCtxChanged, ("fCtxChanged=%#RX64\n", pVCpu->hm.s.fCtxChanged));
8466
8467 /* Store status of the shared guest-host state at the time of VM-entry. */
8468#if HC_ARCH_BITS == 32 && defined(VBOX_WITH_64_BITS_GUESTS)
8469 if (CPUMIsGuestInLongModeEx(pMixedCtx))
8470 {
8471 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActivePending(pVCpu);
8472 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActivePending(pVCpu);
8473 }
8474 else
8475#endif
8476 {
8477 pVmxTransient->fWasGuestDebugStateActive = CPUMIsGuestDebugStateActive(pVCpu);
8478 pVmxTransient->fWasHyperDebugStateActive = CPUMIsHyperDebugStateActive(pVCpu);
8479 }
8480
8481 /*
8482 * Cache the TPR-shadow for checking on every VM-exit if it might have changed.
8483 */
8484 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8485 pVmxTransient->u8GuestTpr = pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR];
8486
8487 PHMGLOBALCPUINFO pCpu = hmR0GetCurrentCpu();
8488 RTCPUID idCurrentCpu = pCpu->idCpu;
8489 if ( pVmxTransient->fUpdateTscOffsettingAndPreemptTimer
8490 || idCurrentCpu != pVCpu->hm.s.idLastCpu)
8491 {
8492 hmR0VmxUpdateTscOffsettingAndPreemptTimer(pVCpu);
8493 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = false;
8494 }
8495
8496 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, true); /* Used for TLB flushing, set this across the world switch. */
8497 hmR0VmxFlushTaggedTlb(pVCpu, pCpu); /* Invalidate the appropriate guest entries from the TLB. */
8498 Assert(idCurrentCpu == pVCpu->hm.s.idLastCpu);
8499 pVCpu->hm.s.vmx.LastError.idCurrentCpu = idCurrentCpu; /* Update the error reporting info. with the current host CPU. */
8500
8501 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatEntry, &pVCpu->hm.s.StatInGC, x);
8502
8503 TMNotifyStartOfExecution(pVCpu); /* Finally, notify TM to resume its clocks as we're about
8504 to start executing. */
8505
8506 /*
8507 * Load the TSC_AUX MSR when we are not intercepting RDTSCP.
8508 */
8509 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDTSCP)
8510 {
8511 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8512 {
8513 bool fMsrUpdated;
8514 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_TSC_AUX);
8515 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX, CPUMGetGuestTscAux(pVCpu), true /* fUpdateHostMsr */,
8516 &fMsrUpdated);
8517 AssertRC(rc2);
8518 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8519 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8520 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8521 }
8522 else
8523 {
8524 hmR0VmxRemoveAutoLoadStoreMsr(pVCpu, MSR_K8_TSC_AUX);
8525 Assert(!pVCpu->hm.s.vmx.cMsrs || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8526 }
8527 }
8528
8529 if (pVM->cpum.ro.GuestFeatures.fIbrs)
8530 {
8531 bool fMsrUpdated;
8532 hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_OTHER_MSRS);
8533 int rc2 = hmR0VmxAddAutoLoadStoreMsr(pVCpu, MSR_IA32_SPEC_CTRL, CPUMGetGuestSpecCtrl(pVCpu), true /* fUpdateHostMsr */,
8534 &fMsrUpdated);
8535 AssertRC(rc2);
8536 Assert(fMsrUpdated || pVCpu->hm.s.vmx.fUpdatedHostMsrs);
8537 /* Finally, mark that all host MSR values are updated so we don't redo it without leaving VT-x. See @bugref{6956}. */
8538 pVCpu->hm.s.vmx.fUpdatedHostMsrs = true;
8539 }
8540
8541#ifdef VBOX_STRICT
8542 hmR0VmxCheckAutoLoadStoreMsrs(pVCpu);
8543 hmR0VmxCheckHostEferMsr(pVCpu);
8544 AssertRC(hmR0VmxCheckVmcsCtls(pVCpu));
8545#endif
8546#ifdef HMVMX_ALWAYS_CHECK_GUEST_STATE
8547 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
8548 if (uInvalidReason != VMX_IGS_REASON_NOT_FOUND)
8549 Log4(("hmR0VmxCheckGuestState returned %#x\n", uInvalidReason));
8550#endif
8551}
8552
8553
8554/**
8555 * Performs some essential restoration of state after running guest code in
8556 * VT-x.
8557 *
8558 * @param pVCpu The cross context virtual CPU structure.
8559 * @param pVmxTransient Pointer to the VMX transient structure.
8560 * @param rcVMRun Return code of VMLAUNCH/VMRESUME.
8561 *
8562 * @remarks Called with interrupts disabled, and returns with interrupts enabled!
8563 *
8564 * @remarks No-long-jump zone!!! This function will however re-enable longjmps
8565 * unconditionally when it is safe to do so.
8566 */
8567static void hmR0VmxPostRunGuest(PVMCPU pVCpu, PVMXTRANSIENT pVmxTransient, int rcVMRun)
8568{
8569 uint64_t const uHostTsc = ASMReadTSC();
8570 Assert(!VMMRZCallRing3IsEnabled(pVCpu));
8571
8572 ASMAtomicWriteBool(&pVCpu->hm.s.fCheckedTLBFlush, false); /* See HMInvalidatePageOnAllVCpus(): used for TLB flushing. */
8573 ASMAtomicIncU32(&pVCpu->hm.s.cWorldSwitchExits); /* Initialized in vmR3CreateUVM(): used for EMT poking. */
8574 pVCpu->hm.s.fCtxChanged = 0; /* Exits/longjmps to ring-3 requires saving the guest state. */
8575 pVmxTransient->fVmcsFieldsRead = 0; /* Transient fields need to be read from the VMCS. */
8576 pVmxTransient->fVectoringPF = false; /* Vectoring page-fault needs to be determined later. */
8577 pVmxTransient->fVectoringDoublePF = false; /* Vectoring double page-fault needs to be determined later. */
8578
8579 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
8580 TMCpuTickSetLastSeen(pVCpu, uHostTsc + pVCpu->hm.s.vmx.u64TscOffset);
8581
8582 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatInGC, &pVCpu->hm.s.StatPreExit, x);
8583 TMNotifyEndOfExecution(pVCpu); /* Notify TM that the guest is no longer running. */
8584 Assert(!ASMIntAreEnabled());
8585 VMCPU_SET_STATE(pVCpu, VMCPUSTATE_STARTED_HM);
8586
8587#if HC_ARCH_BITS == 64
8588 pVCpu->hm.s.vmx.fRestoreHostFlags |= VMX_RESTORE_HOST_REQUIRED; /* Host state messed up by VT-x, we must restore. */
8589#endif
8590#if HC_ARCH_BITS == 32 && defined(VBOX_ENABLE_64_BITS_GUESTS)
8591 /* The 64-on-32 switcher maintains uVmcsState on its own and we need to leave it alone here. */
8592 if (pVCpu->hm.s.vmx.pfnStartVM != VMXR0SwitcherStartVM64)
8593 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8594#else
8595 pVCpu->hm.s.vmx.uVmcsState |= HMVMX_VMCS_STATE_LAUNCHED; /* Use VMRESUME instead of VMLAUNCH in the next run. */
8596#endif
8597#ifdef VBOX_STRICT
8598 hmR0VmxCheckHostEferMsr(pVCpu); /* Verify that VMRUN/VMLAUNCH didn't modify host EFER. */
8599#endif
8600 ASMSetFlags(pVmxTransient->fEFlags); /* Enable interrupts. */
8601
8602 /* Save the basic VM-exit reason. Refer Intel spec. 24.9.1 "Basic VM-exit Information". */
8603 uint32_t uExitReason;
8604 int rc = VMXReadVmcs32(VMX_VMCS32_RO_EXIT_REASON, &uExitReason);
8605 rc |= hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
8606 AssertRC(rc);
8607 pVmxTransient->uExitReason = (uint16_t)VMX_EXIT_REASON_BASIC(uExitReason);
8608 pVmxTransient->fVMEntryFailed = VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uEntryIntInfo);
8609
8610 if (rcVMRun == VINF_SUCCESS)
8611 {
8612 /*
8613 * Update the VM-exit history array here even if the VM-entry failed due to:
8614 * - Invalid guest state.
8615 * - MSR loading.
8616 * - Machine-check event.
8617 *
8618 * In any of the above cases we will still have a "valid" VM-exit reason
8619 * despite @a fVMEntryFailed being false.
8620 *
8621 * See Intel spec. 26.7 "VM-Entry failures during or after loading guest state".
8622 *
8623 * Note! We don't have CS or RIP at this point. Will probably address that later
8624 * by amending the history entry added here.
8625 */
8626 EMHistoryAddExit(pVCpu, EMEXIT_MAKE_FT(EMEXIT_F_KIND_VMX, pVmxTransient->uExitReason & EMEXIT_F_TYPE_MASK),
8627 UINT64_MAX, uHostTsc);
8628
8629 if (!pVmxTransient->fVMEntryFailed)
8630 {
8631 VMMRZCallRing3Enable(pVCpu);
8632
8633 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_CR3));
8634 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_HM_UPDATE_PAE_PDPES));
8635
8636#if defined(HMVMX_ALWAYS_SYNC_FULL_GUEST_STATE) || defined(HMVMX_ALWAYS_SAVE_FULL_GUEST_STATE)
8637 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
8638 AssertRC(rc);
8639#elif defined(HMVMX_ALWAYS_SAVE_GUEST_RFLAGS)
8640 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_RFLAGS);
8641 AssertRC(rc);
8642#else
8643 /*
8644 * Import the guest-interruptibility state always as we need it while evaluating
8645 * injecting events on re-entry.
8646 *
8647 * We don't import CR0 (when Unrestricted guest execution is unavailable) despite
8648 * checking for real-mode while exporting the state because all bits that cause
8649 * mode changes wrt CR0 are intercepted.
8650 */
8651 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_HM_VMX_INT_STATE);
8652 AssertRC(rc);
8653#endif
8654
8655 /*
8656 * Sync the TPR shadow with our APIC state.
8657 */
8658 if ( (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
8659 && pVmxTransient->u8GuestTpr != pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR])
8660 {
8661 rc = APICSetTpr(pVCpu, pVCpu->hm.s.vmx.pbVirtApic[XAPIC_OFF_TPR]);
8662 AssertRC(rc);
8663 ASMAtomicOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
8664 }
8665
8666 return;
8667 }
8668 }
8669 else
8670 {
8671 Log4Func(("VM-entry failure: rcVMRun=%Rrc fVMEntryFailed=%RTbool\n", rcVMRun, pVmxTransient->fVMEntryFailed));
8672 }
8673
8674 VMMRZCallRing3Enable(pVCpu);
8675}
8676
8677
8678/**
8679 * Runs the guest code using VT-x the normal way.
8680 *
8681 * @returns VBox status code.
8682 * @param pVCpu The cross context virtual CPU structure.
8683 * @param pCtx Pointer to the guest-CPU context.
8684 *
8685 * @note Mostly the same as hmR0VmxRunGuestCodeStep().
8686 */
8687static VBOXSTRICTRC hmR0VmxRunGuestCodeNormal(PVMCPU pVCpu, PCPUMCTX pCtx)
8688{
8689 VMXTRANSIENT VmxTransient;
8690 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
8691 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
8692 uint32_t cLoops = 0;
8693
8694 for (;; cLoops++)
8695 {
8696 Assert(!HMR0SuspendPending());
8697 HMVMX_ASSERT_CPU_SAFE();
8698
8699 /* Preparatory work for running guest code, this may force us to return
8700 to ring-3. This bugger disables interrupts on VINF_SUCCESS! */
8701 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
8702 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, false /* fStepping */);
8703 if (rcStrict != VINF_SUCCESS)
8704 break;
8705
8706 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
8707 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
8708 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
8709
8710 /* Restore any residual host-state and save any bits shared between host
8711 and guest into the guest-CPU state. Re-enables interrupts! */
8712 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
8713
8714 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
8715 if (RT_SUCCESS(rcRun))
8716 { /* very likely */ }
8717 else
8718 {
8719 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
8720 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
8721 return rcRun;
8722 }
8723
8724 /* Profile the VM-exit. */
8725 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
8726 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
8727 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
8728 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
8729 HMVMX_START_EXIT_DISPATCH_PROF();
8730
8731 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
8732
8733 /* Handle the VM-exit. */
8734#ifdef HMVMX_USE_FUNCTION_TABLE
8735 rcStrict = g_apfnVMExitHandlers[VmxTransient.uExitReason](pVCpu, pCtx, &VmxTransient);
8736#else
8737 rcStrict = hmR0VmxHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason);
8738#endif
8739 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
8740 if (rcStrict == VINF_SUCCESS)
8741 {
8742 if (cLoops <= pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
8743 continue; /* likely */
8744 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
8745 rcStrict = VINF_EM_RAW_INTERRUPT;
8746 }
8747 break;
8748 }
8749
8750 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
8751 return rcStrict;
8752}
8753
8754
8755
8756/** @name Execution loop for single stepping, DBGF events and expensive Dtrace
8757 * probes.
8758 *
8759 * The following few functions and associated structure contains the bloat
8760 * necessary for providing detailed debug events and dtrace probes as well as
8761 * reliable host side single stepping. This works on the principle of
8762 * "subclassing" the normal execution loop and workers. We replace the loop
8763 * method completely and override selected helpers to add necessary adjustments
8764 * to their core operation.
8765 *
8766 * The goal is to keep the "parent" code lean and mean, so as not to sacrifice
8767 * any performance for debug and analysis features.
8768 *
8769 * @{
8770 */
8771
8772/**
8773 * Transient per-VCPU debug state of VMCS and related info. we save/restore in
8774 * the debug run loop.
8775 */
8776typedef struct VMXRUNDBGSTATE
8777{
8778 /** The RIP we started executing at. This is for detecting that we stepped. */
8779 uint64_t uRipStart;
8780 /** The CS we started executing with. */
8781 uint16_t uCsStart;
8782
8783 /** Whether we've actually modified the 1st execution control field. */
8784 bool fModifiedProcCtls : 1;
8785 /** Whether we've actually modified the 2nd execution control field. */
8786 bool fModifiedProcCtls2 : 1;
8787 /** Whether we've actually modified the exception bitmap. */
8788 bool fModifiedXcptBitmap : 1;
8789
8790 /** We desire the modified the CR0 mask to be cleared. */
8791 bool fClearCr0Mask : 1;
8792 /** We desire the modified the CR4 mask to be cleared. */
8793 bool fClearCr4Mask : 1;
8794 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC. */
8795 uint32_t fCpe1Extra;
8796 /** Stuff we do not want in VMX_VMCS32_CTRL_PROC_EXEC. */
8797 uint32_t fCpe1Unwanted;
8798 /** Stuff we need in VMX_VMCS32_CTRL_PROC_EXEC2. */
8799 uint32_t fCpe2Extra;
8800 /** Extra stuff we need in VMX_VMCS32_CTRL_EXCEPTION_BITMAP. */
8801 uint32_t bmXcptExtra;
8802 /** The sequence number of the Dtrace provider settings the state was
8803 * configured against. */
8804 uint32_t uDtraceSettingsSeqNo;
8805 /** VM-exits to check (one bit per VM-exit). */
8806 uint32_t bmExitsToCheck[3];
8807
8808 /** The initial VMX_VMCS32_CTRL_PROC_EXEC value (helps with restore). */
8809 uint32_t fProcCtlsInitial;
8810 /** The initial VMX_VMCS32_CTRL_PROC_EXEC2 value (helps with restore). */
8811 uint32_t fProcCtls2Initial;
8812 /** The initial VMX_VMCS32_CTRL_EXCEPTION_BITMAP value (helps with restore). */
8813 uint32_t bmXcptInitial;
8814} VMXRUNDBGSTATE;
8815AssertCompileMemberSize(VMXRUNDBGSTATE, bmExitsToCheck, (VMX_EXIT_MAX + 1 + 31) / 32 * 4);
8816typedef VMXRUNDBGSTATE *PVMXRUNDBGSTATE;
8817
8818
8819/**
8820 * Initializes the VMXRUNDBGSTATE structure.
8821 *
8822 * @param pVCpu The cross context virtual CPU structure of the
8823 * calling EMT.
8824 * @param pCtx The CPU register context to go with @a pVCpu.
8825 * @param pDbgState The structure to initialize.
8826 */
8827static void hmR0VmxRunDebugStateInit(PVMCPU pVCpu, PCCPUMCTX pCtx, PVMXRUNDBGSTATE pDbgState)
8828{
8829 pDbgState->uRipStart = pCtx->rip;
8830 pDbgState->uCsStart = pCtx->cs.Sel;
8831
8832 pDbgState->fModifiedProcCtls = false;
8833 pDbgState->fModifiedProcCtls2 = false;
8834 pDbgState->fModifiedXcptBitmap = false;
8835 pDbgState->fClearCr0Mask = false;
8836 pDbgState->fClearCr4Mask = false;
8837 pDbgState->fCpe1Extra = 0;
8838 pDbgState->fCpe1Unwanted = 0;
8839 pDbgState->fCpe2Extra = 0;
8840 pDbgState->bmXcptExtra = 0;
8841 pDbgState->fProcCtlsInitial = pVCpu->hm.s.vmx.u32ProcCtls;
8842 pDbgState->fProcCtls2Initial = pVCpu->hm.s.vmx.u32ProcCtls2;
8843 pDbgState->bmXcptInitial = pVCpu->hm.s.vmx.u32XcptBitmap;
8844}
8845
8846
8847/**
8848 * Updates the VMSC fields with changes requested by @a pDbgState.
8849 *
8850 * This is performed after hmR0VmxPreRunGuestDebugStateUpdate as well
8851 * immediately before executing guest code, i.e. when interrupts are disabled.
8852 * We don't check status codes here as we cannot easily assert or return in the
8853 * latter case.
8854 *
8855 * @param pVCpu The cross context virtual CPU structure.
8856 * @param pDbgState The debug state.
8857 */
8858static void hmR0VmxPreRunGuestDebugStateApply(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState)
8859{
8860 /*
8861 * Ensure desired flags in VMCS control fields are set.
8862 * (Ignoring write failure here, as we're committed and it's just debug extras.)
8863 *
8864 * Note! We load the shadow CR0 & CR4 bits when we flag the clearing, so
8865 * there should be no stale data in pCtx at this point.
8866 */
8867 if ( (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Extra) != pDbgState->fCpe1Extra
8868 || (pVCpu->hm.s.vmx.u32ProcCtls & pDbgState->fCpe1Unwanted))
8869 {
8870 pVCpu->hm.s.vmx.u32ProcCtls |= pDbgState->fCpe1Extra;
8871 pVCpu->hm.s.vmx.u32ProcCtls &= ~pDbgState->fCpe1Unwanted;
8872 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
8873 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls));
8874 pDbgState->fModifiedProcCtls = true;
8875 }
8876
8877 if ((pVCpu->hm.s.vmx.u32ProcCtls2 & pDbgState->fCpe2Extra) != pDbgState->fCpe2Extra)
8878 {
8879 pVCpu->hm.s.vmx.u32ProcCtls2 |= pDbgState->fCpe2Extra;
8880 VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pVCpu->hm.s.vmx.u32ProcCtls2);
8881 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_PROC_EXEC2: %#RX32\n", pVCpu->hm.s.vmx.u32ProcCtls2));
8882 pDbgState->fModifiedProcCtls2 = true;
8883 }
8884
8885 if ((pVCpu->hm.s.vmx.u32XcptBitmap & pDbgState->bmXcptExtra) != pDbgState->bmXcptExtra)
8886 {
8887 pVCpu->hm.s.vmx.u32XcptBitmap |= pDbgState->bmXcptExtra;
8888 VMXWriteVmcs32(VMX_VMCS32_CTRL_EXCEPTION_BITMAP, pVCpu->hm.s.vmx.u32XcptBitmap);
8889 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS32_CTRL_EXCEPTION_BITMAP: %#RX32\n", pVCpu->hm.s.vmx.u32XcptBitmap));
8890 pDbgState->fModifiedXcptBitmap = true;
8891 }
8892
8893 if (pDbgState->fClearCr0Mask && pVCpu->hm.s.vmx.u32CR0Mask != 0)
8894 {
8895 pVCpu->hm.s.vmx.u32CR0Mask = 0;
8896 VMXWriteVmcs32(VMX_VMCS_CTRL_CR0_MASK, 0);
8897 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR0_MASK: 0\n"));
8898 }
8899
8900 if (pDbgState->fClearCr4Mask && pVCpu->hm.s.vmx.u32CR4Mask != 0)
8901 {
8902 pVCpu->hm.s.vmx.u32CR4Mask = 0;
8903 VMXWriteVmcs32(VMX_VMCS_CTRL_CR4_MASK, 0);
8904 Log6(("hmR0VmxRunDebugStateRevert: VMX_VMCS_CTRL_CR4_MASK: 0\n"));
8905 }
8906}
8907
8908
8909static VBOXSTRICTRC hmR0VmxRunDebugStateRevert(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, VBOXSTRICTRC rcStrict)
8910{
8911 /*
8912 * Restore VM-exit control settings as we may not reenter this function the
8913 * next time around.
8914 */
8915 /* We reload the initial value, trigger what we can of recalculations the
8916 next time around. From the looks of things, that's all that's required atm. */
8917 if (pDbgState->fModifiedProcCtls)
8918 {
8919 if (!(pDbgState->fProcCtlsInitial & VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT) && CPUMIsHyperDebugStateActive(pVCpu))
8920 pDbgState->fProcCtlsInitial |= VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT; /* Avoid assertion in hmR0VmxLeave */
8921 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pDbgState->fProcCtlsInitial);
8922 AssertRCReturn(rc2, rc2);
8923 pVCpu->hm.s.vmx.u32ProcCtls = pDbgState->fProcCtlsInitial;
8924 }
8925
8926 /* We're currently the only ones messing with this one, so just restore the
8927 cached value and reload the field. */
8928 if ( pDbgState->fModifiedProcCtls2
8929 && pVCpu->hm.s.vmx.u32ProcCtls2 != pDbgState->fProcCtls2Initial)
8930 {
8931 int rc2 = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC2, pDbgState->fProcCtls2Initial);
8932 AssertRCReturn(rc2, rc2);
8933 pVCpu->hm.s.vmx.u32ProcCtls2 = pDbgState->fProcCtls2Initial;
8934 }
8935
8936 /* If we've modified the exception bitmap, we restore it and trigger
8937 reloading and partial recalculation the next time around. */
8938 if (pDbgState->fModifiedXcptBitmap)
8939 pVCpu->hm.s.vmx.u32XcptBitmap = pDbgState->bmXcptInitial;
8940
8941 return rcStrict;
8942}
8943
8944
8945/**
8946 * Configures VM-exit controls for current DBGF and DTrace settings.
8947 *
8948 * This updates @a pDbgState and the VMCS execution control fields to reflect
8949 * the necessary VM-exits demanded by DBGF and DTrace.
8950 *
8951 * @param pVCpu The cross context virtual CPU structure.
8952 * @param pDbgState The debug state.
8953 * @param pVmxTransient Pointer to the VMX transient structure. May update
8954 * fUpdateTscOffsettingAndPreemptTimer.
8955 */
8956static void hmR0VmxPreRunGuestDebugStateUpdate(PVMCPU pVCpu, PVMXRUNDBGSTATE pDbgState, PVMXTRANSIENT pVmxTransient)
8957{
8958 /*
8959 * Take down the dtrace serial number so we can spot changes.
8960 */
8961 pDbgState->uDtraceSettingsSeqNo = VBOXVMM_GET_SETTINGS_SEQ_NO();
8962 ASMCompilerBarrier();
8963
8964 /*
8965 * We'll rebuild most of the middle block of data members (holding the
8966 * current settings) as we go along here, so start by clearing it all.
8967 */
8968 pDbgState->bmXcptExtra = 0;
8969 pDbgState->fCpe1Extra = 0;
8970 pDbgState->fCpe1Unwanted = 0;
8971 pDbgState->fCpe2Extra = 0;
8972 for (unsigned i = 0; i < RT_ELEMENTS(pDbgState->bmExitsToCheck); i++)
8973 pDbgState->bmExitsToCheck[i] = 0;
8974
8975 /*
8976 * Software interrupts (INT XXh) - no idea how to trigger these...
8977 */
8978 PVM pVM = pVCpu->CTX_SUFF(pVM);
8979 if ( DBGF_IS_EVENT_ENABLED(pVM, DBGFEVENT_INTERRUPT_SOFTWARE)
8980 || VBOXVMM_INT_SOFTWARE_ENABLED())
8981 {
8982 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
8983 }
8984
8985 /*
8986 * INT3 breakpoints - triggered by #BP exceptions.
8987 */
8988 if (pVM->dbgf.ro.cEnabledInt3Breakpoints > 0)
8989 pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
8990
8991 /*
8992 * Exception bitmap and XCPT events+probes.
8993 */
8994 for (int iXcpt = 0; iXcpt < (DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST + 1); iXcpt++)
8995 if (DBGF_IS_EVENT_ENABLED(pVM, (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + iXcpt)))
8996 pDbgState->bmXcptExtra |= RT_BIT_32(iXcpt);
8997
8998 if (VBOXVMM_XCPT_DE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DE);
8999 if (VBOXVMM_XCPT_DB_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DB);
9000 if (VBOXVMM_XCPT_BP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BP);
9001 if (VBOXVMM_XCPT_OF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_OF);
9002 if (VBOXVMM_XCPT_BR_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_BR);
9003 if (VBOXVMM_XCPT_UD_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_UD);
9004 if (VBOXVMM_XCPT_NM_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NM);
9005 if (VBOXVMM_XCPT_DF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_DF);
9006 if (VBOXVMM_XCPT_TS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_TS);
9007 if (VBOXVMM_XCPT_NP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_NP);
9008 if (VBOXVMM_XCPT_SS_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SS);
9009 if (VBOXVMM_XCPT_GP_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_GP);
9010 if (VBOXVMM_XCPT_PF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_PF);
9011 if (VBOXVMM_XCPT_MF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_MF);
9012 if (VBOXVMM_XCPT_AC_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_AC);
9013 if (VBOXVMM_XCPT_XF_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_XF);
9014 if (VBOXVMM_XCPT_VE_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_VE);
9015 if (VBOXVMM_XCPT_SX_ENABLED()) pDbgState->bmXcptExtra |= RT_BIT_32(X86_XCPT_SX);
9016
9017 if (pDbgState->bmXcptExtra)
9018 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XCPT_OR_NMI);
9019
9020 /*
9021 * Process events and probes for VM-exits, making sure we get the wanted VM-exits.
9022 *
9023 * Note! This is the reverse of what hmR0VmxHandleExitDtraceEvents does.
9024 * So, when adding/changing/removing please don't forget to update it.
9025 *
9026 * Some of the macros are picking up local variables to save horizontal space,
9027 * (being able to see it in a table is the lesser evil here).
9028 */
9029#define IS_EITHER_ENABLED(a_pVM, a_EventSubName) \
9030 ( DBGF_IS_EVENT_ENABLED(a_pVM, RT_CONCAT(DBGFEVENT_, a_EventSubName)) \
9031 || RT_CONCAT3(VBOXVMM_, a_EventSubName, _ENABLED)() )
9032#define SET_ONLY_XBM_IF_EITHER_EN(a_EventSubName, a_uExit) \
9033 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9034 { AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9035 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9036 } else do { } while (0)
9037#define SET_CPE1_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec) \
9038 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9039 { \
9040 (pDbgState)->fCpe1Extra |= (a_fCtrlProcExec); \
9041 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9042 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9043 } else do { } while (0)
9044#define SET_CPEU_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fUnwantedCtrlProcExec) \
9045 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9046 { \
9047 (pDbgState)->fCpe1Unwanted |= (a_fUnwantedCtrlProcExec); \
9048 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9049 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9050 } else do { } while (0)
9051#define SET_CPE2_XBM_IF_EITHER_EN(a_EventSubName, a_uExit, a_fCtrlProcExec2) \
9052 if (IS_EITHER_ENABLED(pVM, a_EventSubName)) \
9053 { \
9054 (pDbgState)->fCpe2Extra |= (a_fCtrlProcExec2); \
9055 AssertCompile((unsigned)(a_uExit) < sizeof(pDbgState->bmExitsToCheck) * 8); \
9056 ASMBitSet((pDbgState)->bmExitsToCheck, a_uExit); \
9057 } else do { } while (0)
9058
9059 SET_ONLY_XBM_IF_EITHER_EN(EXIT_TASK_SWITCH, VMX_EXIT_TASK_SWITCH); /* unconditional */
9060 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_VIOLATION, VMX_EXIT_EPT_VIOLATION); /* unconditional */
9061 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_EPT_MISCONFIG, VMX_EXIT_EPT_MISCONFIG); /* unconditional (unless #VE) */
9062 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_ACCESS, VMX_EXIT_APIC_ACCESS); /* feature dependent, nothing to enable here */
9063 SET_ONLY_XBM_IF_EITHER_EN(EXIT_VMX_VAPIC_WRITE, VMX_EXIT_APIC_WRITE); /* feature dependent, nothing to enable here */
9064
9065 SET_ONLY_XBM_IF_EITHER_EN(INSTR_CPUID, VMX_EXIT_CPUID); /* unconditional */
9066 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CPUID, VMX_EXIT_CPUID);
9067 SET_ONLY_XBM_IF_EITHER_EN(INSTR_GETSEC, VMX_EXIT_GETSEC); /* unconditional */
9068 SET_ONLY_XBM_IF_EITHER_EN( EXIT_GETSEC, VMX_EXIT_GETSEC);
9069 SET_CPE1_XBM_IF_EITHER_EN(INSTR_HALT, VMX_EXIT_HLT, VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT); /* paranoia */
9070 SET_ONLY_XBM_IF_EITHER_EN( EXIT_HALT, VMX_EXIT_HLT);
9071 SET_ONLY_XBM_IF_EITHER_EN(INSTR_INVD, VMX_EXIT_INVD); /* unconditional */
9072 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVD, VMX_EXIT_INVD);
9073 SET_CPE1_XBM_IF_EITHER_EN(INSTR_INVLPG, VMX_EXIT_INVLPG, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9074 SET_ONLY_XBM_IF_EITHER_EN( EXIT_INVLPG, VMX_EXIT_INVLPG);
9075 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDPMC, VMX_EXIT_RDPMC, VMX_VMCS_CTRL_PROC_EXEC_RDPMC_EXIT);
9076 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDPMC, VMX_EXIT_RDPMC);
9077 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSC, VMX_EXIT_RDTSC, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9078 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSC, VMX_EXIT_RDTSC);
9079 SET_ONLY_XBM_IF_EITHER_EN(INSTR_RSM, VMX_EXIT_RSM); /* unconditional */
9080 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RSM, VMX_EXIT_RSM);
9081 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMM_CALL, VMX_EXIT_VMCALL); /* unconditional */
9082 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMM_CALL, VMX_EXIT_VMCALL);
9083 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMCLEAR, VMX_EXIT_VMCLEAR); /* unconditional */
9084 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMCLEAR, VMX_EXIT_VMCLEAR);
9085 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH); /* unconditional */
9086 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMLAUNCH, VMX_EXIT_VMLAUNCH);
9087 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRLD, VMX_EXIT_VMPTRLD); /* unconditional */
9088 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRLD, VMX_EXIT_VMPTRLD);
9089 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMPTRST, VMX_EXIT_VMPTRST); /* unconditional */
9090 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMPTRST, VMX_EXIT_VMPTRST);
9091 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMREAD, VMX_EXIT_VMREAD); /* unconditional */
9092 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMREAD, VMX_EXIT_VMREAD);
9093 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMRESUME, VMX_EXIT_VMRESUME); /* unconditional */
9094 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMRESUME, VMX_EXIT_VMRESUME);
9095 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMWRITE, VMX_EXIT_VMWRITE); /* unconditional */
9096 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMWRITE, VMX_EXIT_VMWRITE);
9097 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXOFF, VMX_EXIT_VMXOFF); /* unconditional */
9098 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXOFF, VMX_EXIT_VMXOFF);
9099 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMXON, VMX_EXIT_VMXON); /* unconditional */
9100 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMXON, VMX_EXIT_VMXON);
9101
9102 if ( IS_EITHER_ENABLED(pVM, INSTR_CRX_READ)
9103 || IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9104 {
9105 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
9106 | CPUMCTX_EXTRN_CR4
9107 | CPUMCTX_EXTRN_APIC_TPR);
9108 AssertRC(rc);
9109
9110#if 0 /** @todo fix me */
9111 pDbgState->fClearCr0Mask = true;
9112 pDbgState->fClearCr4Mask = true;
9113#endif
9114 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_READ))
9115 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_STORE_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_STORE_EXIT;
9116 if (IS_EITHER_ENABLED(pVM, INSTR_CRX_WRITE))
9117 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_CR3_LOAD_EXIT | VMX_VMCS_CTRL_PROC_EXEC_CR8_LOAD_EXIT;
9118 pDbgState->fCpe1Unwanted |= VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW; /* risky? */
9119 /* Note! We currently don't use VMX_VMCS32_CTRL_CR3_TARGET_COUNT. It would
9120 require clearing here and in the loop if we start using it. */
9121 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_CRX);
9122 }
9123 else
9124 {
9125 if (pDbgState->fClearCr0Mask)
9126 {
9127 pDbgState->fClearCr0Mask = false;
9128 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
9129 }
9130 if (pDbgState->fClearCr4Mask)
9131 {
9132 pDbgState->fClearCr4Mask = false;
9133 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
9134 }
9135 }
9136 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_READ, VMX_EXIT_MOV_CRX);
9137 SET_ONLY_XBM_IF_EITHER_EN( EXIT_CRX_WRITE, VMX_EXIT_MOV_CRX);
9138
9139 if ( IS_EITHER_ENABLED(pVM, INSTR_DRX_READ)
9140 || IS_EITHER_ENABLED(pVM, INSTR_DRX_WRITE))
9141 {
9142 /** @todo later, need to fix handler as it assumes this won't usually happen. */
9143 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_MOV_DRX);
9144 }
9145 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_READ, VMX_EXIT_MOV_DRX);
9146 SET_ONLY_XBM_IF_EITHER_EN( EXIT_DRX_WRITE, VMX_EXIT_MOV_DRX);
9147
9148 SET_CPEU_XBM_IF_EITHER_EN(INSTR_RDMSR, VMX_EXIT_RDMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS); /* risky clearing this? */
9149 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDMSR, VMX_EXIT_RDMSR);
9150 SET_CPEU_XBM_IF_EITHER_EN(INSTR_WRMSR, VMX_EXIT_WRMSR, VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS);
9151 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WRMSR, VMX_EXIT_WRMSR);
9152 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MWAIT, VMX_EXIT_MWAIT, VMX_VMCS_CTRL_PROC_EXEC_MWAIT_EXIT); /* paranoia */
9153 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MWAIT, VMX_EXIT_MWAIT);
9154 SET_CPE1_XBM_IF_EITHER_EN(INSTR_MONITOR, VMX_EXIT_MONITOR, VMX_VMCS_CTRL_PROC_EXEC_MONITOR_EXIT); /* paranoia */
9155 SET_ONLY_XBM_IF_EITHER_EN( EXIT_MONITOR, VMX_EXIT_MONITOR);
9156#if 0 /** @todo too slow, fix handler. */
9157 SET_CPE1_XBM_IF_EITHER_EN(INSTR_PAUSE, VMX_EXIT_PAUSE, VMX_VMCS_CTRL_PROC_EXEC_PAUSE_EXIT);
9158#endif
9159 SET_ONLY_XBM_IF_EITHER_EN( EXIT_PAUSE, VMX_EXIT_PAUSE);
9160
9161 if ( IS_EITHER_ENABLED(pVM, INSTR_SGDT)
9162 || IS_EITHER_ENABLED(pVM, INSTR_SIDT)
9163 || IS_EITHER_ENABLED(pVM, INSTR_LGDT)
9164 || IS_EITHER_ENABLED(pVM, INSTR_LIDT))
9165 {
9166 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9167 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_XDTR_ACCESS);
9168 }
9169 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SGDT, VMX_EXIT_XDTR_ACCESS);
9170 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SIDT, VMX_EXIT_XDTR_ACCESS);
9171 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LGDT, VMX_EXIT_XDTR_ACCESS);
9172 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LIDT, VMX_EXIT_XDTR_ACCESS);
9173
9174 if ( IS_EITHER_ENABLED(pVM, INSTR_SLDT)
9175 || IS_EITHER_ENABLED(pVM, INSTR_STR)
9176 || IS_EITHER_ENABLED(pVM, INSTR_LLDT)
9177 || IS_EITHER_ENABLED(pVM, INSTR_LTR))
9178 {
9179 pDbgState->fCpe2Extra |= VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT;
9180 ASMBitSet(pDbgState->bmExitsToCheck, VMX_EXIT_TR_ACCESS);
9181 }
9182 SET_ONLY_XBM_IF_EITHER_EN( EXIT_SLDT, VMX_EXIT_TR_ACCESS);
9183 SET_ONLY_XBM_IF_EITHER_EN( EXIT_STR, VMX_EXIT_TR_ACCESS);
9184 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LLDT, VMX_EXIT_TR_ACCESS);
9185 SET_ONLY_XBM_IF_EITHER_EN( EXIT_LTR, VMX_EXIT_TR_ACCESS);
9186
9187 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVEPT, VMX_EXIT_INVEPT); /* unconditional */
9188 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVEPT, VMX_EXIT_INVEPT);
9189 SET_CPE1_XBM_IF_EITHER_EN(INSTR_RDTSCP, VMX_EXIT_RDTSCP, VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT);
9190 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDTSCP, VMX_EXIT_RDTSCP);
9191 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_INVVPID, VMX_EXIT_INVVPID); /* unconditional */
9192 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVVPID, VMX_EXIT_INVVPID);
9193 SET_CPE2_XBM_IF_EITHER_EN(INSTR_WBINVD, VMX_EXIT_WBINVD, VMX_VMCS_CTRL_PROC_EXEC2_WBINVD_EXIT);
9194 SET_ONLY_XBM_IF_EITHER_EN( EXIT_WBINVD, VMX_EXIT_WBINVD);
9195 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSETBV, VMX_EXIT_XSETBV); /* unconditional */
9196 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XSETBV, VMX_EXIT_XSETBV);
9197 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDRAND, VMX_EXIT_RDRAND, VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT);
9198 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDRAND, VMX_EXIT_RDRAND);
9199 SET_CPE1_XBM_IF_EITHER_EN(INSTR_VMX_INVPCID, VMX_EXIT_INVPCID, VMX_VMCS_CTRL_PROC_EXEC_INVLPG_EXIT);
9200 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_INVPCID, VMX_EXIT_INVPCID);
9201 SET_ONLY_XBM_IF_EITHER_EN(INSTR_VMX_VMFUNC, VMX_EXIT_VMFUNC); /* unconditional for the current setup */
9202 SET_ONLY_XBM_IF_EITHER_EN( EXIT_VMX_VMFUNC, VMX_EXIT_VMFUNC);
9203 SET_CPE2_XBM_IF_EITHER_EN(INSTR_RDSEED, VMX_EXIT_RDSEED, VMX_VMCS_CTRL_PROC_EXEC2_RDSEED_EXIT);
9204 SET_ONLY_XBM_IF_EITHER_EN( EXIT_RDSEED, VMX_EXIT_RDSEED);
9205 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XSAVES, VMX_EXIT_XSAVES); /* unconditional (enabled by host, guest cfg) */
9206 SET_ONLY_XBM_IF_EITHER_EN(EXIT_XSAVES, VMX_EXIT_XSAVES);
9207 SET_ONLY_XBM_IF_EITHER_EN(INSTR_XRSTORS, VMX_EXIT_XRSTORS); /* unconditional (enabled by host, guest cfg) */
9208 SET_ONLY_XBM_IF_EITHER_EN( EXIT_XRSTORS, VMX_EXIT_XRSTORS);
9209
9210#undef IS_EITHER_ENABLED
9211#undef SET_ONLY_XBM_IF_EITHER_EN
9212#undef SET_CPE1_XBM_IF_EITHER_EN
9213#undef SET_CPEU_XBM_IF_EITHER_EN
9214#undef SET_CPE2_XBM_IF_EITHER_EN
9215
9216 /*
9217 * Sanitize the control stuff.
9218 */
9219 pDbgState->fCpe2Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls2.n.allowed1;
9220 if (pDbgState->fCpe2Extra)
9221 pDbgState->fCpe1Extra |= VMX_VMCS_CTRL_PROC_EXEC_USE_SECONDARY_EXEC_CTRL;
9222 pDbgState->fCpe1Extra &= pVM->hm.s.vmx.Msrs.VmxProcCtls.n.allowed1;
9223 pDbgState->fCpe1Unwanted &= ~pVM->hm.s.vmx.Msrs.VmxProcCtls.n.disallowed0;
9224 if (pVCpu->hm.s.fDebugWantRdTscExit != RT_BOOL(pDbgState->fCpe1Extra & VMX_VMCS_CTRL_PROC_EXEC_RDTSC_EXIT))
9225 {
9226 pVCpu->hm.s.fDebugWantRdTscExit ^= true;
9227 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
9228 }
9229
9230 Log6(("HM: debug state: cpe1=%#RX32 cpeu=%#RX32 cpe2=%#RX32%s%s\n",
9231 pDbgState->fCpe1Extra, pDbgState->fCpe1Unwanted, pDbgState->fCpe2Extra,
9232 pDbgState->fClearCr0Mask ? " clr-cr0" : "",
9233 pDbgState->fClearCr4Mask ? " clr-cr4" : ""));
9234}
9235
9236
9237/**
9238 * Fires off DBGF events and dtrace probes for a VM-exit, when it's
9239 * appropriate.
9240 *
9241 * The caller has checked the VM-exit against the
9242 * VMXRUNDBGSTATE::bmExitsToCheck bitmap. The caller has checked for NMIs
9243 * already, so we don't have to do that either.
9244 *
9245 * @returns Strict VBox status code (i.e. informational status codes too).
9246 * @param pVCpu The cross context virtual CPU structure.
9247 * @param pMixedCtx Pointer to the guest-CPU context.
9248 * @param pVmxTransient Pointer to the VMX-transient structure.
9249 * @param uExitReason The VM-exit reason.
9250 *
9251 * @remarks The name of this function is displayed by dtrace, so keep it short
9252 * and to the point. No longer than 33 chars long, please.
9253 */
9254static VBOXSTRICTRC hmR0VmxHandleExitDtraceEvents(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9255 uint32_t uExitReason)
9256{
9257 /*
9258 * Translate the event into a DBGF event (enmEvent + uEventArg) and at the
9259 * same time check whether any corresponding Dtrace event is enabled (fDtrace).
9260 *
9261 * Note! This is the reverse operation of what hmR0VmxPreRunGuestDebugStateUpdate
9262 * does. Must add/change/remove both places. Same ordering, please.
9263 *
9264 * Added/removed events must also be reflected in the next section
9265 * where we dispatch dtrace events.
9266 */
9267 bool fDtrace1 = false;
9268 bool fDtrace2 = false;
9269 DBGFEVENTTYPE enmEvent1 = DBGFEVENT_END;
9270 DBGFEVENTTYPE enmEvent2 = DBGFEVENT_END;
9271 uint32_t uEventArg = 0;
9272#define SET_EXIT(a_EventSubName) \
9273 do { \
9274 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9275 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9276 } while (0)
9277#define SET_BOTH(a_EventSubName) \
9278 do { \
9279 enmEvent1 = RT_CONCAT(DBGFEVENT_INSTR_, a_EventSubName); \
9280 enmEvent2 = RT_CONCAT(DBGFEVENT_EXIT_, a_EventSubName); \
9281 fDtrace1 = RT_CONCAT3(VBOXVMM_INSTR_, a_EventSubName, _ENABLED)(); \
9282 fDtrace2 = RT_CONCAT3(VBOXVMM_EXIT_, a_EventSubName, _ENABLED)(); \
9283 } while (0)
9284 switch (uExitReason)
9285 {
9286 case VMX_EXIT_MTF:
9287 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9288
9289 case VMX_EXIT_XCPT_OR_NMI:
9290 {
9291 uint8_t const idxVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
9292 switch (VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo))
9293 {
9294 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
9295 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT:
9296 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT:
9297 if (idxVector <= (unsigned)(DBGFEVENT_XCPT_LAST - DBGFEVENT_XCPT_FIRST))
9298 {
9299 if (VMX_EXIT_INTERRUPTION_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uExitIntInfo))
9300 {
9301 hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
9302 uEventArg = pVmxTransient->uExitIntErrorCode;
9303 }
9304 enmEvent1 = (DBGFEVENTTYPE)(DBGFEVENT_XCPT_FIRST + idxVector);
9305 switch (enmEvent1)
9306 {
9307 case DBGFEVENT_XCPT_DE: fDtrace1 = VBOXVMM_XCPT_DE_ENABLED(); break;
9308 case DBGFEVENT_XCPT_DB: fDtrace1 = VBOXVMM_XCPT_DB_ENABLED(); break;
9309 case DBGFEVENT_XCPT_BP: fDtrace1 = VBOXVMM_XCPT_BP_ENABLED(); break;
9310 case DBGFEVENT_XCPT_OF: fDtrace1 = VBOXVMM_XCPT_OF_ENABLED(); break;
9311 case DBGFEVENT_XCPT_BR: fDtrace1 = VBOXVMM_XCPT_BR_ENABLED(); break;
9312 case DBGFEVENT_XCPT_UD: fDtrace1 = VBOXVMM_XCPT_UD_ENABLED(); break;
9313 case DBGFEVENT_XCPT_NM: fDtrace1 = VBOXVMM_XCPT_NM_ENABLED(); break;
9314 case DBGFEVENT_XCPT_DF: fDtrace1 = VBOXVMM_XCPT_DF_ENABLED(); break;
9315 case DBGFEVENT_XCPT_TS: fDtrace1 = VBOXVMM_XCPT_TS_ENABLED(); break;
9316 case DBGFEVENT_XCPT_NP: fDtrace1 = VBOXVMM_XCPT_NP_ENABLED(); break;
9317 case DBGFEVENT_XCPT_SS: fDtrace1 = VBOXVMM_XCPT_SS_ENABLED(); break;
9318 case DBGFEVENT_XCPT_GP: fDtrace1 = VBOXVMM_XCPT_GP_ENABLED(); break;
9319 case DBGFEVENT_XCPT_PF: fDtrace1 = VBOXVMM_XCPT_PF_ENABLED(); break;
9320 case DBGFEVENT_XCPT_MF: fDtrace1 = VBOXVMM_XCPT_MF_ENABLED(); break;
9321 case DBGFEVENT_XCPT_AC: fDtrace1 = VBOXVMM_XCPT_AC_ENABLED(); break;
9322 case DBGFEVENT_XCPT_XF: fDtrace1 = VBOXVMM_XCPT_XF_ENABLED(); break;
9323 case DBGFEVENT_XCPT_VE: fDtrace1 = VBOXVMM_XCPT_VE_ENABLED(); break;
9324 case DBGFEVENT_XCPT_SX: fDtrace1 = VBOXVMM_XCPT_SX_ENABLED(); break;
9325 default: break;
9326 }
9327 }
9328 else
9329 AssertFailed();
9330 break;
9331
9332 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_INT:
9333 uEventArg = idxVector;
9334 enmEvent1 = DBGFEVENT_INTERRUPT_SOFTWARE;
9335 fDtrace1 = VBOXVMM_INT_SOFTWARE_ENABLED();
9336 break;
9337 }
9338 break;
9339 }
9340
9341 case VMX_EXIT_TRIPLE_FAULT:
9342 enmEvent1 = DBGFEVENT_TRIPLE_FAULT;
9343 //fDtrace1 = VBOXVMM_EXIT_TRIPLE_FAULT_ENABLED();
9344 break;
9345 case VMX_EXIT_TASK_SWITCH: SET_EXIT(TASK_SWITCH); break;
9346 case VMX_EXIT_EPT_VIOLATION: SET_EXIT(VMX_EPT_VIOLATION); break;
9347 case VMX_EXIT_EPT_MISCONFIG: SET_EXIT(VMX_EPT_MISCONFIG); break;
9348 case VMX_EXIT_APIC_ACCESS: SET_EXIT(VMX_VAPIC_ACCESS); break;
9349 case VMX_EXIT_APIC_WRITE: SET_EXIT(VMX_VAPIC_WRITE); break;
9350
9351 /* Instruction specific VM-exits: */
9352 case VMX_EXIT_CPUID: SET_BOTH(CPUID); break;
9353 case VMX_EXIT_GETSEC: SET_BOTH(GETSEC); break;
9354 case VMX_EXIT_HLT: SET_BOTH(HALT); break;
9355 case VMX_EXIT_INVD: SET_BOTH(INVD); break;
9356 case VMX_EXIT_INVLPG: SET_BOTH(INVLPG); break;
9357 case VMX_EXIT_RDPMC: SET_BOTH(RDPMC); break;
9358 case VMX_EXIT_RDTSC: SET_BOTH(RDTSC); break;
9359 case VMX_EXIT_RSM: SET_BOTH(RSM); break;
9360 case VMX_EXIT_VMCALL: SET_BOTH(VMM_CALL); break;
9361 case VMX_EXIT_VMCLEAR: SET_BOTH(VMX_VMCLEAR); break;
9362 case VMX_EXIT_VMLAUNCH: SET_BOTH(VMX_VMLAUNCH); break;
9363 case VMX_EXIT_VMPTRLD: SET_BOTH(VMX_VMPTRLD); break;
9364 case VMX_EXIT_VMPTRST: SET_BOTH(VMX_VMPTRST); break;
9365 case VMX_EXIT_VMREAD: SET_BOTH(VMX_VMREAD); break;
9366 case VMX_EXIT_VMRESUME: SET_BOTH(VMX_VMRESUME); break;
9367 case VMX_EXIT_VMWRITE: SET_BOTH(VMX_VMWRITE); break;
9368 case VMX_EXIT_VMXOFF: SET_BOTH(VMX_VMXOFF); break;
9369 case VMX_EXIT_VMXON: SET_BOTH(VMX_VMXON); break;
9370 case VMX_EXIT_MOV_CRX:
9371 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9372 if (VMX_EXIT_QUAL_CRX_ACCESS(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_CRX_ACCESS_READ)
9373 SET_BOTH(CRX_READ);
9374 else
9375 SET_BOTH(CRX_WRITE);
9376 uEventArg = VMX_EXIT_QUAL_CRX_REGISTER(pVmxTransient->uExitQualification);
9377 break;
9378 case VMX_EXIT_MOV_DRX:
9379 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9380 if ( VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification)
9381 == VMX_EXIT_QUAL_DRX_DIRECTION_READ)
9382 SET_BOTH(DRX_READ);
9383 else
9384 SET_BOTH(DRX_WRITE);
9385 uEventArg = VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification);
9386 break;
9387 case VMX_EXIT_RDMSR: SET_BOTH(RDMSR); break;
9388 case VMX_EXIT_WRMSR: SET_BOTH(WRMSR); break;
9389 case VMX_EXIT_MWAIT: SET_BOTH(MWAIT); break;
9390 case VMX_EXIT_MONITOR: SET_BOTH(MONITOR); break;
9391 case VMX_EXIT_PAUSE: SET_BOTH(PAUSE); break;
9392 case VMX_EXIT_XDTR_ACCESS:
9393 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9394 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_XDTR_INSINFO_INSTR_ID))
9395 {
9396 case VMX_XDTR_INSINFO_II_SGDT: SET_BOTH(SGDT); break;
9397 case VMX_XDTR_INSINFO_II_SIDT: SET_BOTH(SIDT); break;
9398 case VMX_XDTR_INSINFO_II_LGDT: SET_BOTH(LGDT); break;
9399 case VMX_XDTR_INSINFO_II_LIDT: SET_BOTH(LIDT); break;
9400 }
9401 break;
9402
9403 case VMX_EXIT_TR_ACCESS:
9404 hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
9405 switch (RT_BF_GET(pVmxTransient->ExitInstrInfo.u, VMX_YYTR_INSINFO_INSTR_ID))
9406 {
9407 case VMX_YYTR_INSINFO_II_SLDT: SET_BOTH(SLDT); break;
9408 case VMX_YYTR_INSINFO_II_STR: SET_BOTH(STR); break;
9409 case VMX_YYTR_INSINFO_II_LLDT: SET_BOTH(LLDT); break;
9410 case VMX_YYTR_INSINFO_II_LTR: SET_BOTH(LTR); break;
9411 }
9412 break;
9413
9414 case VMX_EXIT_INVEPT: SET_BOTH(VMX_INVEPT); break;
9415 case VMX_EXIT_RDTSCP: SET_BOTH(RDTSCP); break;
9416 case VMX_EXIT_INVVPID: SET_BOTH(VMX_INVVPID); break;
9417 case VMX_EXIT_WBINVD: SET_BOTH(WBINVD); break;
9418 case VMX_EXIT_XSETBV: SET_BOTH(XSETBV); break;
9419 case VMX_EXIT_RDRAND: SET_BOTH(RDRAND); break;
9420 case VMX_EXIT_INVPCID: SET_BOTH(VMX_INVPCID); break;
9421 case VMX_EXIT_VMFUNC: SET_BOTH(VMX_VMFUNC); break;
9422 case VMX_EXIT_RDSEED: SET_BOTH(RDSEED); break;
9423 case VMX_EXIT_XSAVES: SET_BOTH(XSAVES); break;
9424 case VMX_EXIT_XRSTORS: SET_BOTH(XRSTORS); break;
9425
9426 /* Events that aren't relevant at this point. */
9427 case VMX_EXIT_EXT_INT:
9428 case VMX_EXIT_INT_WINDOW:
9429 case VMX_EXIT_NMI_WINDOW:
9430 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9431 case VMX_EXIT_PREEMPT_TIMER:
9432 case VMX_EXIT_IO_INSTR:
9433 break;
9434
9435 /* Errors and unexpected events. */
9436 case VMX_EXIT_INIT_SIGNAL:
9437 case VMX_EXIT_SIPI:
9438 case VMX_EXIT_IO_SMI:
9439 case VMX_EXIT_SMI:
9440 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9441 case VMX_EXIT_ERR_MSR_LOAD:
9442 case VMX_EXIT_ERR_MACHINE_CHECK:
9443 break;
9444
9445 default:
9446 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9447 break;
9448 }
9449#undef SET_BOTH
9450#undef SET_EXIT
9451
9452 /*
9453 * Dtrace tracepoints go first. We do them here at once so we don't
9454 * have to copy the guest state saving and stuff a few dozen times.
9455 * Down side is that we've got to repeat the switch, though this time
9456 * we use enmEvent since the probes are a subset of what DBGF does.
9457 */
9458 if (fDtrace1 || fDtrace2)
9459 {
9460 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9461 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9462 switch (enmEvent1)
9463 {
9464 /** @todo consider which extra parameters would be helpful for each probe. */
9465 case DBGFEVENT_END: break;
9466 case DBGFEVENT_XCPT_DE: VBOXVMM_XCPT_DE(pVCpu, pMixedCtx); break;
9467 case DBGFEVENT_XCPT_DB: VBOXVMM_XCPT_DB(pVCpu, pMixedCtx, pMixedCtx->dr[6]); break;
9468 case DBGFEVENT_XCPT_BP: VBOXVMM_XCPT_BP(pVCpu, pMixedCtx); break;
9469 case DBGFEVENT_XCPT_OF: VBOXVMM_XCPT_OF(pVCpu, pMixedCtx); break;
9470 case DBGFEVENT_XCPT_BR: VBOXVMM_XCPT_BR(pVCpu, pMixedCtx); break;
9471 case DBGFEVENT_XCPT_UD: VBOXVMM_XCPT_UD(pVCpu, pMixedCtx); break;
9472 case DBGFEVENT_XCPT_NM: VBOXVMM_XCPT_NM(pVCpu, pMixedCtx); break;
9473 case DBGFEVENT_XCPT_DF: VBOXVMM_XCPT_DF(pVCpu, pMixedCtx); break;
9474 case DBGFEVENT_XCPT_TS: VBOXVMM_XCPT_TS(pVCpu, pMixedCtx, uEventArg); break;
9475 case DBGFEVENT_XCPT_NP: VBOXVMM_XCPT_NP(pVCpu, pMixedCtx, uEventArg); break;
9476 case DBGFEVENT_XCPT_SS: VBOXVMM_XCPT_SS(pVCpu, pMixedCtx, uEventArg); break;
9477 case DBGFEVENT_XCPT_GP: VBOXVMM_XCPT_GP(pVCpu, pMixedCtx, uEventArg); break;
9478 case DBGFEVENT_XCPT_PF: VBOXVMM_XCPT_PF(pVCpu, pMixedCtx, uEventArg, pMixedCtx->cr2); break;
9479 case DBGFEVENT_XCPT_MF: VBOXVMM_XCPT_MF(pVCpu, pMixedCtx); break;
9480 case DBGFEVENT_XCPT_AC: VBOXVMM_XCPT_AC(pVCpu, pMixedCtx); break;
9481 case DBGFEVENT_XCPT_XF: VBOXVMM_XCPT_XF(pVCpu, pMixedCtx); break;
9482 case DBGFEVENT_XCPT_VE: VBOXVMM_XCPT_VE(pVCpu, pMixedCtx); break;
9483 case DBGFEVENT_XCPT_SX: VBOXVMM_XCPT_SX(pVCpu, pMixedCtx, uEventArg); break;
9484 case DBGFEVENT_INTERRUPT_SOFTWARE: VBOXVMM_INT_SOFTWARE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9485 case DBGFEVENT_INSTR_CPUID: VBOXVMM_INSTR_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9486 case DBGFEVENT_INSTR_GETSEC: VBOXVMM_INSTR_GETSEC(pVCpu, pMixedCtx); break;
9487 case DBGFEVENT_INSTR_HALT: VBOXVMM_INSTR_HALT(pVCpu, pMixedCtx); break;
9488 case DBGFEVENT_INSTR_INVD: VBOXVMM_INSTR_INVD(pVCpu, pMixedCtx); break;
9489 case DBGFEVENT_INSTR_INVLPG: VBOXVMM_INSTR_INVLPG(pVCpu, pMixedCtx); break;
9490 case DBGFEVENT_INSTR_RDPMC: VBOXVMM_INSTR_RDPMC(pVCpu, pMixedCtx); break;
9491 case DBGFEVENT_INSTR_RDTSC: VBOXVMM_INSTR_RDTSC(pVCpu, pMixedCtx); break;
9492 case DBGFEVENT_INSTR_RSM: VBOXVMM_INSTR_RSM(pVCpu, pMixedCtx); break;
9493 case DBGFEVENT_INSTR_CRX_READ: VBOXVMM_INSTR_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9494 case DBGFEVENT_INSTR_CRX_WRITE: VBOXVMM_INSTR_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9495 case DBGFEVENT_INSTR_DRX_READ: VBOXVMM_INSTR_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9496 case DBGFEVENT_INSTR_DRX_WRITE: VBOXVMM_INSTR_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9497 case DBGFEVENT_INSTR_RDMSR: VBOXVMM_INSTR_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9498 case DBGFEVENT_INSTR_WRMSR: VBOXVMM_INSTR_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9499 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9500 case DBGFEVENT_INSTR_MWAIT: VBOXVMM_INSTR_MWAIT(pVCpu, pMixedCtx); break;
9501 case DBGFEVENT_INSTR_MONITOR: VBOXVMM_INSTR_MONITOR(pVCpu, pMixedCtx); break;
9502 case DBGFEVENT_INSTR_PAUSE: VBOXVMM_INSTR_PAUSE(pVCpu, pMixedCtx); break;
9503 case DBGFEVENT_INSTR_SGDT: VBOXVMM_INSTR_SGDT(pVCpu, pMixedCtx); break;
9504 case DBGFEVENT_INSTR_SIDT: VBOXVMM_INSTR_SIDT(pVCpu, pMixedCtx); break;
9505 case DBGFEVENT_INSTR_LGDT: VBOXVMM_INSTR_LGDT(pVCpu, pMixedCtx); break;
9506 case DBGFEVENT_INSTR_LIDT: VBOXVMM_INSTR_LIDT(pVCpu, pMixedCtx); break;
9507 case DBGFEVENT_INSTR_SLDT: VBOXVMM_INSTR_SLDT(pVCpu, pMixedCtx); break;
9508 case DBGFEVENT_INSTR_STR: VBOXVMM_INSTR_STR(pVCpu, pMixedCtx); break;
9509 case DBGFEVENT_INSTR_LLDT: VBOXVMM_INSTR_LLDT(pVCpu, pMixedCtx); break;
9510 case DBGFEVENT_INSTR_LTR: VBOXVMM_INSTR_LTR(pVCpu, pMixedCtx); break;
9511 case DBGFEVENT_INSTR_RDTSCP: VBOXVMM_INSTR_RDTSCP(pVCpu, pMixedCtx); break;
9512 case DBGFEVENT_INSTR_WBINVD: VBOXVMM_INSTR_WBINVD(pVCpu, pMixedCtx); break;
9513 case DBGFEVENT_INSTR_XSETBV: VBOXVMM_INSTR_XSETBV(pVCpu, pMixedCtx); break;
9514 case DBGFEVENT_INSTR_RDRAND: VBOXVMM_INSTR_RDRAND(pVCpu, pMixedCtx); break;
9515 case DBGFEVENT_INSTR_RDSEED: VBOXVMM_INSTR_RDSEED(pVCpu, pMixedCtx); break;
9516 case DBGFEVENT_INSTR_XSAVES: VBOXVMM_INSTR_XSAVES(pVCpu, pMixedCtx); break;
9517 case DBGFEVENT_INSTR_XRSTORS: VBOXVMM_INSTR_XRSTORS(pVCpu, pMixedCtx); break;
9518 case DBGFEVENT_INSTR_VMM_CALL: VBOXVMM_INSTR_VMM_CALL(pVCpu, pMixedCtx); break;
9519 case DBGFEVENT_INSTR_VMX_VMCLEAR: VBOXVMM_INSTR_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9520 case DBGFEVENT_INSTR_VMX_VMLAUNCH: VBOXVMM_INSTR_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9521 case DBGFEVENT_INSTR_VMX_VMPTRLD: VBOXVMM_INSTR_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9522 case DBGFEVENT_INSTR_VMX_VMPTRST: VBOXVMM_INSTR_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9523 case DBGFEVENT_INSTR_VMX_VMREAD: VBOXVMM_INSTR_VMX_VMREAD(pVCpu, pMixedCtx); break;
9524 case DBGFEVENT_INSTR_VMX_VMRESUME: VBOXVMM_INSTR_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9525 case DBGFEVENT_INSTR_VMX_VMWRITE: VBOXVMM_INSTR_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9526 case DBGFEVENT_INSTR_VMX_VMXOFF: VBOXVMM_INSTR_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9527 case DBGFEVENT_INSTR_VMX_VMXON: VBOXVMM_INSTR_VMX_VMXON(pVCpu, pMixedCtx); break;
9528 case DBGFEVENT_INSTR_VMX_INVEPT: VBOXVMM_INSTR_VMX_INVEPT(pVCpu, pMixedCtx); break;
9529 case DBGFEVENT_INSTR_VMX_INVVPID: VBOXVMM_INSTR_VMX_INVVPID(pVCpu, pMixedCtx); break;
9530 case DBGFEVENT_INSTR_VMX_INVPCID: VBOXVMM_INSTR_VMX_INVPCID(pVCpu, pMixedCtx); break;
9531 case DBGFEVENT_INSTR_VMX_VMFUNC: VBOXVMM_INSTR_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9532 default: AssertMsgFailed(("enmEvent1=%d uExitReason=%d\n", enmEvent1, uExitReason)); break;
9533 }
9534 switch (enmEvent2)
9535 {
9536 /** @todo consider which extra parameters would be helpful for each probe. */
9537 case DBGFEVENT_END: break;
9538 case DBGFEVENT_EXIT_TASK_SWITCH: VBOXVMM_EXIT_TASK_SWITCH(pVCpu, pMixedCtx); break;
9539 case DBGFEVENT_EXIT_CPUID: VBOXVMM_EXIT_CPUID(pVCpu, pMixedCtx, pMixedCtx->eax, pMixedCtx->ecx); break;
9540 case DBGFEVENT_EXIT_GETSEC: VBOXVMM_EXIT_GETSEC(pVCpu, pMixedCtx); break;
9541 case DBGFEVENT_EXIT_HALT: VBOXVMM_EXIT_HALT(pVCpu, pMixedCtx); break;
9542 case DBGFEVENT_EXIT_INVD: VBOXVMM_EXIT_INVD(pVCpu, pMixedCtx); break;
9543 case DBGFEVENT_EXIT_INVLPG: VBOXVMM_EXIT_INVLPG(pVCpu, pMixedCtx); break;
9544 case DBGFEVENT_EXIT_RDPMC: VBOXVMM_EXIT_RDPMC(pVCpu, pMixedCtx); break;
9545 case DBGFEVENT_EXIT_RDTSC: VBOXVMM_EXIT_RDTSC(pVCpu, pMixedCtx); break;
9546 case DBGFEVENT_EXIT_RSM: VBOXVMM_EXIT_RSM(pVCpu, pMixedCtx); break;
9547 case DBGFEVENT_EXIT_CRX_READ: VBOXVMM_EXIT_CRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9548 case DBGFEVENT_EXIT_CRX_WRITE: VBOXVMM_EXIT_CRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9549 case DBGFEVENT_EXIT_DRX_READ: VBOXVMM_EXIT_DRX_READ(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9550 case DBGFEVENT_EXIT_DRX_WRITE: VBOXVMM_EXIT_DRX_WRITE(pVCpu, pMixedCtx, (uint8_t)uEventArg); break;
9551 case DBGFEVENT_EXIT_RDMSR: VBOXVMM_EXIT_RDMSR(pVCpu, pMixedCtx, pMixedCtx->ecx); break;
9552 case DBGFEVENT_EXIT_WRMSR: VBOXVMM_EXIT_WRMSR(pVCpu, pMixedCtx, pMixedCtx->ecx,
9553 RT_MAKE_U64(pMixedCtx->eax, pMixedCtx->edx)); break;
9554 case DBGFEVENT_EXIT_MWAIT: VBOXVMM_EXIT_MWAIT(pVCpu, pMixedCtx); break;
9555 case DBGFEVENT_EXIT_MONITOR: VBOXVMM_EXIT_MONITOR(pVCpu, pMixedCtx); break;
9556 case DBGFEVENT_EXIT_PAUSE: VBOXVMM_EXIT_PAUSE(pVCpu, pMixedCtx); break;
9557 case DBGFEVENT_EXIT_SGDT: VBOXVMM_EXIT_SGDT(pVCpu, pMixedCtx); break;
9558 case DBGFEVENT_EXIT_SIDT: VBOXVMM_EXIT_SIDT(pVCpu, pMixedCtx); break;
9559 case DBGFEVENT_EXIT_LGDT: VBOXVMM_EXIT_LGDT(pVCpu, pMixedCtx); break;
9560 case DBGFEVENT_EXIT_LIDT: VBOXVMM_EXIT_LIDT(pVCpu, pMixedCtx); break;
9561 case DBGFEVENT_EXIT_SLDT: VBOXVMM_EXIT_SLDT(pVCpu, pMixedCtx); break;
9562 case DBGFEVENT_EXIT_STR: VBOXVMM_EXIT_STR(pVCpu, pMixedCtx); break;
9563 case DBGFEVENT_EXIT_LLDT: VBOXVMM_EXIT_LLDT(pVCpu, pMixedCtx); break;
9564 case DBGFEVENT_EXIT_LTR: VBOXVMM_EXIT_LTR(pVCpu, pMixedCtx); break;
9565 case DBGFEVENT_EXIT_RDTSCP: VBOXVMM_EXIT_RDTSCP(pVCpu, pMixedCtx); break;
9566 case DBGFEVENT_EXIT_WBINVD: VBOXVMM_EXIT_WBINVD(pVCpu, pMixedCtx); break;
9567 case DBGFEVENT_EXIT_XSETBV: VBOXVMM_EXIT_XSETBV(pVCpu, pMixedCtx); break;
9568 case DBGFEVENT_EXIT_RDRAND: VBOXVMM_EXIT_RDRAND(pVCpu, pMixedCtx); break;
9569 case DBGFEVENT_EXIT_RDSEED: VBOXVMM_EXIT_RDSEED(pVCpu, pMixedCtx); break;
9570 case DBGFEVENT_EXIT_XSAVES: VBOXVMM_EXIT_XSAVES(pVCpu, pMixedCtx); break;
9571 case DBGFEVENT_EXIT_XRSTORS: VBOXVMM_EXIT_XRSTORS(pVCpu, pMixedCtx); break;
9572 case DBGFEVENT_EXIT_VMM_CALL: VBOXVMM_EXIT_VMM_CALL(pVCpu, pMixedCtx); break;
9573 case DBGFEVENT_EXIT_VMX_VMCLEAR: VBOXVMM_EXIT_VMX_VMCLEAR(pVCpu, pMixedCtx); break;
9574 case DBGFEVENT_EXIT_VMX_VMLAUNCH: VBOXVMM_EXIT_VMX_VMLAUNCH(pVCpu, pMixedCtx); break;
9575 case DBGFEVENT_EXIT_VMX_VMPTRLD: VBOXVMM_EXIT_VMX_VMPTRLD(pVCpu, pMixedCtx); break;
9576 case DBGFEVENT_EXIT_VMX_VMPTRST: VBOXVMM_EXIT_VMX_VMPTRST(pVCpu, pMixedCtx); break;
9577 case DBGFEVENT_EXIT_VMX_VMREAD: VBOXVMM_EXIT_VMX_VMREAD(pVCpu, pMixedCtx); break;
9578 case DBGFEVENT_EXIT_VMX_VMRESUME: VBOXVMM_EXIT_VMX_VMRESUME(pVCpu, pMixedCtx); break;
9579 case DBGFEVENT_EXIT_VMX_VMWRITE: VBOXVMM_EXIT_VMX_VMWRITE(pVCpu, pMixedCtx); break;
9580 case DBGFEVENT_EXIT_VMX_VMXOFF: VBOXVMM_EXIT_VMX_VMXOFF(pVCpu, pMixedCtx); break;
9581 case DBGFEVENT_EXIT_VMX_VMXON: VBOXVMM_EXIT_VMX_VMXON(pVCpu, pMixedCtx); break;
9582 case DBGFEVENT_EXIT_VMX_INVEPT: VBOXVMM_EXIT_VMX_INVEPT(pVCpu, pMixedCtx); break;
9583 case DBGFEVENT_EXIT_VMX_INVVPID: VBOXVMM_EXIT_VMX_INVVPID(pVCpu, pMixedCtx); break;
9584 case DBGFEVENT_EXIT_VMX_INVPCID: VBOXVMM_EXIT_VMX_INVPCID(pVCpu, pMixedCtx); break;
9585 case DBGFEVENT_EXIT_VMX_VMFUNC: VBOXVMM_EXIT_VMX_VMFUNC(pVCpu, pMixedCtx); break;
9586 case DBGFEVENT_EXIT_VMX_EPT_MISCONFIG: VBOXVMM_EXIT_VMX_EPT_MISCONFIG(pVCpu, pMixedCtx); break;
9587 case DBGFEVENT_EXIT_VMX_EPT_VIOLATION: VBOXVMM_EXIT_VMX_EPT_VIOLATION(pVCpu, pMixedCtx); break;
9588 case DBGFEVENT_EXIT_VMX_VAPIC_ACCESS: VBOXVMM_EXIT_VMX_VAPIC_ACCESS(pVCpu, pMixedCtx); break;
9589 case DBGFEVENT_EXIT_VMX_VAPIC_WRITE: VBOXVMM_EXIT_VMX_VAPIC_WRITE(pVCpu, pMixedCtx); break;
9590 default: AssertMsgFailed(("enmEvent2=%d uExitReason=%d\n", enmEvent2, uExitReason)); break;
9591 }
9592 }
9593
9594 /*
9595 * Fire of the DBGF event, if enabled (our check here is just a quick one,
9596 * the DBGF call will do a full check).
9597 *
9598 * Note! DBGF sets DBGFEVENT_INTERRUPT_SOFTWARE in the bitmap.
9599 * Note! If we have to events, we prioritize the first, i.e. the instruction
9600 * one, in order to avoid event nesting.
9601 */
9602 PVM pVM = pVCpu->CTX_SUFF(pVM);
9603 if ( enmEvent1 != DBGFEVENT_END
9604 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent1))
9605 {
9606 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent1, uEventArg, DBGFEVENTCTX_HM);
9607 if (rcStrict != VINF_SUCCESS)
9608 return rcStrict;
9609 }
9610 else if ( enmEvent2 != DBGFEVENT_END
9611 && DBGF_IS_EVENT_ENABLED(pVM, enmEvent2))
9612 {
9613 VBOXSTRICTRC rcStrict = DBGFEventGenericWithArg(pVM, pVCpu, enmEvent2, uEventArg, DBGFEVENTCTX_HM);
9614 if (rcStrict != VINF_SUCCESS)
9615 return rcStrict;
9616 }
9617
9618 return VINF_SUCCESS;
9619}
9620
9621
9622/**
9623 * Single-stepping VM-exit filtering.
9624 *
9625 * This is preprocessing the VM-exits and deciding whether we've gotten far
9626 * enough to return VINF_EM_DBG_STEPPED already. If not, normal VM-exit
9627 * handling is performed.
9628 *
9629 * @returns Strict VBox status code (i.e. informational status codes too).
9630 * @param pVCpu The cross context virtual CPU structure of the calling EMT.
9631 * @param pMixedCtx Pointer to the guest-CPU context. The data may be
9632 * out-of-sync. Make sure to update the required
9633 * fields before using them.
9634 * @param pVmxTransient Pointer to the VMX-transient structure.
9635 * @param uExitReason The VM-exit reason.
9636 * @param pDbgState The debug state.
9637 */
9638DECLINLINE(VBOXSTRICTRC) hmR0VmxRunDebugHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient,
9639 uint32_t uExitReason, PVMXRUNDBGSTATE pDbgState)
9640{
9641 /*
9642 * Expensive (saves context) generic dtrace VM-exit probe.
9643 */
9644 if (!VBOXVMM_R0_HMVMX_VMEXIT_ENABLED())
9645 { /* more likely */ }
9646 else
9647 {
9648 hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
9649 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
9650 AssertRC(rc);
9651 VBOXVMM_R0_HMVMX_VMEXIT(pVCpu, pMixedCtx, pVmxTransient->uExitReason, pVmxTransient->uExitQualification);
9652 }
9653
9654 /*
9655 * Check for host NMI, just to get that out of the way.
9656 */
9657 if (uExitReason != VMX_EXIT_XCPT_OR_NMI)
9658 { /* normally likely */ }
9659 else
9660 {
9661 int rc2 = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
9662 AssertRCReturn(rc2, rc2);
9663 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
9664 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
9665 return hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient);
9666 }
9667
9668 /*
9669 * Check for single stepping event if we're stepping.
9670 */
9671 if (pVCpu->hm.s.fSingleInstruction)
9672 {
9673 switch (uExitReason)
9674 {
9675 case VMX_EXIT_MTF:
9676 return hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient);
9677
9678 /* Various events: */
9679 case VMX_EXIT_XCPT_OR_NMI:
9680 case VMX_EXIT_EXT_INT:
9681 case VMX_EXIT_TRIPLE_FAULT:
9682 case VMX_EXIT_INT_WINDOW:
9683 case VMX_EXIT_NMI_WINDOW:
9684 case VMX_EXIT_TASK_SWITCH:
9685 case VMX_EXIT_TPR_BELOW_THRESHOLD:
9686 case VMX_EXIT_APIC_ACCESS:
9687 case VMX_EXIT_EPT_VIOLATION:
9688 case VMX_EXIT_EPT_MISCONFIG:
9689 case VMX_EXIT_PREEMPT_TIMER:
9690
9691 /* Instruction specific VM-exits: */
9692 case VMX_EXIT_CPUID:
9693 case VMX_EXIT_GETSEC:
9694 case VMX_EXIT_HLT:
9695 case VMX_EXIT_INVD:
9696 case VMX_EXIT_INVLPG:
9697 case VMX_EXIT_RDPMC:
9698 case VMX_EXIT_RDTSC:
9699 case VMX_EXIT_RSM:
9700 case VMX_EXIT_VMCALL:
9701 case VMX_EXIT_VMCLEAR:
9702 case VMX_EXIT_VMLAUNCH:
9703 case VMX_EXIT_VMPTRLD:
9704 case VMX_EXIT_VMPTRST:
9705 case VMX_EXIT_VMREAD:
9706 case VMX_EXIT_VMRESUME:
9707 case VMX_EXIT_VMWRITE:
9708 case VMX_EXIT_VMXOFF:
9709 case VMX_EXIT_VMXON:
9710 case VMX_EXIT_MOV_CRX:
9711 case VMX_EXIT_MOV_DRX:
9712 case VMX_EXIT_IO_INSTR:
9713 case VMX_EXIT_RDMSR:
9714 case VMX_EXIT_WRMSR:
9715 case VMX_EXIT_MWAIT:
9716 case VMX_EXIT_MONITOR:
9717 case VMX_EXIT_PAUSE:
9718 case VMX_EXIT_XDTR_ACCESS:
9719 case VMX_EXIT_TR_ACCESS:
9720 case VMX_EXIT_INVEPT:
9721 case VMX_EXIT_RDTSCP:
9722 case VMX_EXIT_INVVPID:
9723 case VMX_EXIT_WBINVD:
9724 case VMX_EXIT_XSETBV:
9725 case VMX_EXIT_RDRAND:
9726 case VMX_EXIT_INVPCID:
9727 case VMX_EXIT_VMFUNC:
9728 case VMX_EXIT_RDSEED:
9729 case VMX_EXIT_XSAVES:
9730 case VMX_EXIT_XRSTORS:
9731 {
9732 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9733 | CPUMCTX_EXTRN_CS);
9734 AssertRCReturn(rc, rc);
9735 if ( pMixedCtx->rip != pDbgState->uRipStart
9736 || pMixedCtx->cs.Sel != pDbgState->uCsStart)
9737 return VINF_EM_DBG_STEPPED;
9738 break;
9739 }
9740
9741 /* Errors and unexpected events: */
9742 case VMX_EXIT_INIT_SIGNAL:
9743 case VMX_EXIT_SIPI:
9744 case VMX_EXIT_IO_SMI:
9745 case VMX_EXIT_SMI:
9746 case VMX_EXIT_ERR_INVALID_GUEST_STATE:
9747 case VMX_EXIT_ERR_MSR_LOAD:
9748 case VMX_EXIT_ERR_MACHINE_CHECK:
9749 case VMX_EXIT_APIC_WRITE: /* Some talk about this being fault like, so I guess we must process it? */
9750 break;
9751
9752 default:
9753 AssertMsgFailed(("Unexpected VM-exit=%#x\n", uExitReason));
9754 break;
9755 }
9756 }
9757
9758 /*
9759 * Check for debugger event breakpoints and dtrace probes.
9760 */
9761 if ( uExitReason < RT_ELEMENTS(pDbgState->bmExitsToCheck) * 32U
9762 && ASMBitTest(pDbgState->bmExitsToCheck, uExitReason) )
9763 {
9764 VBOXSTRICTRC rcStrict = hmR0VmxHandleExitDtraceEvents(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9765 if (rcStrict != VINF_SUCCESS)
9766 return rcStrict;
9767 }
9768
9769 /*
9770 * Normal processing.
9771 */
9772#ifdef HMVMX_USE_FUNCTION_TABLE
9773 return g_apfnVMExitHandlers[uExitReason](pVCpu, pMixedCtx, pVmxTransient);
9774#else
9775 return hmR0VmxHandleExit(pVCpu, pMixedCtx, pVmxTransient, uExitReason);
9776#endif
9777}
9778
9779
9780/**
9781 * Single steps guest code using VT-x.
9782 *
9783 * @returns Strict VBox status code (i.e. informational status codes too).
9784 * @param pVCpu The cross context virtual CPU structure.
9785 * @param pCtx Pointer to the guest-CPU context.
9786 *
9787 * @note Mostly the same as hmR0VmxRunGuestCodeNormal().
9788 */
9789static VBOXSTRICTRC hmR0VmxRunGuestCodeDebug(PVMCPU pVCpu, PCPUMCTX pCtx)
9790{
9791 VMXTRANSIENT VmxTransient;
9792 VmxTransient.fUpdateTscOffsettingAndPreemptTimer = true;
9793
9794 /* Set HMCPU indicators. */
9795 bool const fSavedSingleInstruction = pVCpu->hm.s.fSingleInstruction;
9796 pVCpu->hm.s.fSingleInstruction = pVCpu->hm.s.fSingleInstruction || DBGFIsStepping(pVCpu);
9797 pVCpu->hm.s.fDebugWantRdTscExit = false;
9798 pVCpu->hm.s.fUsingDebugLoop = true;
9799
9800 /* State we keep to help modify and later restore the VMCS fields we alter, and for detecting steps. */
9801 VMXRUNDBGSTATE DbgState;
9802 hmR0VmxRunDebugStateInit(pVCpu, pCtx, &DbgState);
9803 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9804
9805 /*
9806 * The loop.
9807 */
9808 VBOXSTRICTRC rcStrict = VERR_INTERNAL_ERROR_5;
9809 for (uint32_t cLoops = 0; ; cLoops++)
9810 {
9811 Assert(!HMR0SuspendPending());
9812 HMVMX_ASSERT_CPU_SAFE();
9813 bool fStepping = pVCpu->hm.s.fSingleInstruction;
9814
9815 /*
9816 * Preparatory work for running guest code, this may force us to return
9817 * to ring-3. This bugger disables interrupts on VINF_SUCCESS!
9818 */
9819 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatEntry, x);
9820 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Set up execute controls the next to can respond to. */
9821 rcStrict = hmR0VmxPreRunGuest(pVCpu, pCtx, &VmxTransient, fStepping);
9822 if (rcStrict != VINF_SUCCESS)
9823 break;
9824
9825 hmR0VmxPreRunGuestCommitted(pVCpu, pCtx, &VmxTransient);
9826 hmR0VmxPreRunGuestDebugStateApply(pVCpu, &DbgState); /* Override any obnoxious code in the above two calls. */
9827
9828 /*
9829 * Now we can run the guest code.
9830 */
9831 int rcRun = hmR0VmxRunGuest(pVCpu, pCtx);
9832
9833 /* The guest-CPU context is now outdated, 'pCtx' is to be treated as 'pMixedCtx' from this point on!!! */
9834
9835 /*
9836 * Restore any residual host-state and save any bits shared between host
9837 * and guest into the guest-CPU state. Re-enables interrupts!
9838 */
9839 hmR0VmxPostRunGuest(pVCpu, &VmxTransient, rcRun);
9840
9841 /* Check for errors with running the VM (VMLAUNCH/VMRESUME). */
9842 if (RT_SUCCESS(rcRun))
9843 { /* very likely */ }
9844 else
9845 {
9846 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatPreExit, x);
9847 hmR0VmxReportWorldSwitchError(pVCpu, rcRun, pCtx, &VmxTransient);
9848 return rcRun;
9849 }
9850
9851 /* Profile the VM-exit. */
9852 AssertMsg(VmxTransient.uExitReason <= VMX_EXIT_MAX, ("%#x\n", VmxTransient.uExitReason));
9853 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitAll);
9854 STAM_COUNTER_INC(&pVCpu->hm.s.paStatExitReasonR0[VmxTransient.uExitReason & MASK_EXITREASON_STAT]);
9855 STAM_PROFILE_ADV_STOP_START(&pVCpu->hm.s.StatPreExit, &pVCpu->hm.s.StatExitHandling, x);
9856 HMVMX_START_EXIT_DISPATCH_PROF();
9857
9858 VBOXVMM_R0_HMVMX_VMEXIT_NOCTX(pVCpu, pCtx, VmxTransient.uExitReason);
9859
9860 /*
9861 * Handle the VM-exit - we quit earlier on certain VM-exits, see hmR0VmxHandleExitDebug().
9862 */
9863 rcStrict = hmR0VmxRunDebugHandleExit(pVCpu, pCtx, &VmxTransient, VmxTransient.uExitReason, &DbgState);
9864 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitHandling, x);
9865 if (rcStrict != VINF_SUCCESS)
9866 break;
9867 if (cLoops > pVCpu->CTX_SUFF(pVM)->hm.s.cMaxResumeLoops)
9868 {
9869 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchMaxResumeLoops);
9870 rcStrict = VINF_EM_RAW_INTERRUPT;
9871 break;
9872 }
9873
9874 /*
9875 * Stepping: Did the RIP change, if so, consider it a single step.
9876 * Otherwise, make sure one of the TFs gets set.
9877 */
9878 if (fStepping)
9879 {
9880 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
9881 | CPUMCTX_EXTRN_CS);
9882 AssertRC(rc);
9883 if ( pCtx->rip != DbgState.uRipStart
9884 || pCtx->cs.Sel != DbgState.uCsStart)
9885 {
9886 rcStrict = VINF_EM_DBG_STEPPED;
9887 break;
9888 }
9889 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
9890 }
9891
9892 /*
9893 * Update when dtrace settings changes (DBGF kicks us, so no need to check).
9894 */
9895 if (VBOXVMM_GET_SETTINGS_SEQ_NO() != DbgState.uDtraceSettingsSeqNo)
9896 hmR0VmxPreRunGuestDebugStateUpdate(pVCpu, &DbgState, &VmxTransient);
9897 }
9898
9899 /*
9900 * Clear the X86_EFL_TF if necessary.
9901 */
9902 if (pVCpu->hm.s.fClearTrapFlag)
9903 {
9904 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RFLAGS);
9905 AssertRC(rc);
9906 pVCpu->hm.s.fClearTrapFlag = false;
9907 pCtx->eflags.Bits.u1TF = 0;
9908 }
9909 /** @todo there seems to be issues with the resume flag when the monitor trap
9910 * flag is pending without being used. Seen early in bios init when
9911 * accessing APIC page in protected mode. */
9912
9913 /*
9914 * Restore VM-exit control settings as we may not reenter this function the
9915 * next time around.
9916 */
9917 rcStrict = hmR0VmxRunDebugStateRevert(pVCpu, &DbgState, rcStrict);
9918
9919 /* Restore HMCPU indicators. */
9920 pVCpu->hm.s.fUsingDebugLoop = false;
9921 pVCpu->hm.s.fDebugWantRdTscExit = false;
9922 pVCpu->hm.s.fSingleInstruction = fSavedSingleInstruction;
9923
9924 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatEntry, x);
9925 return rcStrict;
9926}
9927
9928
9929/** @} */
9930
9931
9932/**
9933 * Checks if any expensive dtrace probes are enabled and we should go to the
9934 * debug loop.
9935 *
9936 * @returns true if we should use debug loop, false if not.
9937 */
9938static bool hmR0VmxAnyExpensiveProbesEnabled(void)
9939{
9940 /* It's probably faster to OR the raw 32-bit counter variables together.
9941 Since the variables are in an array and the probes are next to one
9942 another (more or less), we have good locality. So, better read
9943 eight-nine cache lines ever time and only have one conditional, than
9944 128+ conditionals, right? */
9945 return ( VBOXVMM_R0_HMVMX_VMEXIT_ENABLED_RAW() /* expensive too due to context */
9946 | VBOXVMM_XCPT_DE_ENABLED_RAW()
9947 | VBOXVMM_XCPT_DB_ENABLED_RAW()
9948 | VBOXVMM_XCPT_BP_ENABLED_RAW()
9949 | VBOXVMM_XCPT_OF_ENABLED_RAW()
9950 | VBOXVMM_XCPT_BR_ENABLED_RAW()
9951 | VBOXVMM_XCPT_UD_ENABLED_RAW()
9952 | VBOXVMM_XCPT_NM_ENABLED_RAW()
9953 | VBOXVMM_XCPT_DF_ENABLED_RAW()
9954 | VBOXVMM_XCPT_TS_ENABLED_RAW()
9955 | VBOXVMM_XCPT_NP_ENABLED_RAW()
9956 | VBOXVMM_XCPT_SS_ENABLED_RAW()
9957 | VBOXVMM_XCPT_GP_ENABLED_RAW()
9958 | VBOXVMM_XCPT_PF_ENABLED_RAW()
9959 | VBOXVMM_XCPT_MF_ENABLED_RAW()
9960 | VBOXVMM_XCPT_AC_ENABLED_RAW()
9961 | VBOXVMM_XCPT_XF_ENABLED_RAW()
9962 | VBOXVMM_XCPT_VE_ENABLED_RAW()
9963 | VBOXVMM_XCPT_SX_ENABLED_RAW()
9964 | VBOXVMM_INT_SOFTWARE_ENABLED_RAW()
9965 | VBOXVMM_INT_HARDWARE_ENABLED_RAW()
9966 ) != 0
9967 || ( VBOXVMM_INSTR_HALT_ENABLED_RAW()
9968 | VBOXVMM_INSTR_MWAIT_ENABLED_RAW()
9969 | VBOXVMM_INSTR_MONITOR_ENABLED_RAW()
9970 | VBOXVMM_INSTR_CPUID_ENABLED_RAW()
9971 | VBOXVMM_INSTR_INVD_ENABLED_RAW()
9972 | VBOXVMM_INSTR_WBINVD_ENABLED_RAW()
9973 | VBOXVMM_INSTR_INVLPG_ENABLED_RAW()
9974 | VBOXVMM_INSTR_RDTSC_ENABLED_RAW()
9975 | VBOXVMM_INSTR_RDTSCP_ENABLED_RAW()
9976 | VBOXVMM_INSTR_RDPMC_ENABLED_RAW()
9977 | VBOXVMM_INSTR_RDMSR_ENABLED_RAW()
9978 | VBOXVMM_INSTR_WRMSR_ENABLED_RAW()
9979 | VBOXVMM_INSTR_CRX_READ_ENABLED_RAW()
9980 | VBOXVMM_INSTR_CRX_WRITE_ENABLED_RAW()
9981 | VBOXVMM_INSTR_DRX_READ_ENABLED_RAW()
9982 | VBOXVMM_INSTR_DRX_WRITE_ENABLED_RAW()
9983 | VBOXVMM_INSTR_PAUSE_ENABLED_RAW()
9984 | VBOXVMM_INSTR_XSETBV_ENABLED_RAW()
9985 | VBOXVMM_INSTR_SIDT_ENABLED_RAW()
9986 | VBOXVMM_INSTR_LIDT_ENABLED_RAW()
9987 | VBOXVMM_INSTR_SGDT_ENABLED_RAW()
9988 | VBOXVMM_INSTR_LGDT_ENABLED_RAW()
9989 | VBOXVMM_INSTR_SLDT_ENABLED_RAW()
9990 | VBOXVMM_INSTR_LLDT_ENABLED_RAW()
9991 | VBOXVMM_INSTR_STR_ENABLED_RAW()
9992 | VBOXVMM_INSTR_LTR_ENABLED_RAW()
9993 | VBOXVMM_INSTR_GETSEC_ENABLED_RAW()
9994 | VBOXVMM_INSTR_RSM_ENABLED_RAW()
9995 | VBOXVMM_INSTR_RDRAND_ENABLED_RAW()
9996 | VBOXVMM_INSTR_RDSEED_ENABLED_RAW()
9997 | VBOXVMM_INSTR_XSAVES_ENABLED_RAW()
9998 | VBOXVMM_INSTR_XRSTORS_ENABLED_RAW()
9999 | VBOXVMM_INSTR_VMM_CALL_ENABLED_RAW()
10000 | VBOXVMM_INSTR_VMX_VMCLEAR_ENABLED_RAW()
10001 | VBOXVMM_INSTR_VMX_VMLAUNCH_ENABLED_RAW()
10002 | VBOXVMM_INSTR_VMX_VMPTRLD_ENABLED_RAW()
10003 | VBOXVMM_INSTR_VMX_VMPTRST_ENABLED_RAW()
10004 | VBOXVMM_INSTR_VMX_VMREAD_ENABLED_RAW()
10005 | VBOXVMM_INSTR_VMX_VMRESUME_ENABLED_RAW()
10006 | VBOXVMM_INSTR_VMX_VMWRITE_ENABLED_RAW()
10007 | VBOXVMM_INSTR_VMX_VMXOFF_ENABLED_RAW()
10008 | VBOXVMM_INSTR_VMX_VMXON_ENABLED_RAW()
10009 | VBOXVMM_INSTR_VMX_VMFUNC_ENABLED_RAW()
10010 | VBOXVMM_INSTR_VMX_INVEPT_ENABLED_RAW()
10011 | VBOXVMM_INSTR_VMX_INVVPID_ENABLED_RAW()
10012 | VBOXVMM_INSTR_VMX_INVPCID_ENABLED_RAW()
10013 ) != 0
10014 || ( VBOXVMM_EXIT_TASK_SWITCH_ENABLED_RAW()
10015 | VBOXVMM_EXIT_HALT_ENABLED_RAW()
10016 | VBOXVMM_EXIT_MWAIT_ENABLED_RAW()
10017 | VBOXVMM_EXIT_MONITOR_ENABLED_RAW()
10018 | VBOXVMM_EXIT_CPUID_ENABLED_RAW()
10019 | VBOXVMM_EXIT_INVD_ENABLED_RAW()
10020 | VBOXVMM_EXIT_WBINVD_ENABLED_RAW()
10021 | VBOXVMM_EXIT_INVLPG_ENABLED_RAW()
10022 | VBOXVMM_EXIT_RDTSC_ENABLED_RAW()
10023 | VBOXVMM_EXIT_RDTSCP_ENABLED_RAW()
10024 | VBOXVMM_EXIT_RDPMC_ENABLED_RAW()
10025 | VBOXVMM_EXIT_RDMSR_ENABLED_RAW()
10026 | VBOXVMM_EXIT_WRMSR_ENABLED_RAW()
10027 | VBOXVMM_EXIT_CRX_READ_ENABLED_RAW()
10028 | VBOXVMM_EXIT_CRX_WRITE_ENABLED_RAW()
10029 | VBOXVMM_EXIT_DRX_READ_ENABLED_RAW()
10030 | VBOXVMM_EXIT_DRX_WRITE_ENABLED_RAW()
10031 | VBOXVMM_EXIT_PAUSE_ENABLED_RAW()
10032 | VBOXVMM_EXIT_XSETBV_ENABLED_RAW()
10033 | VBOXVMM_EXIT_SIDT_ENABLED_RAW()
10034 | VBOXVMM_EXIT_LIDT_ENABLED_RAW()
10035 | VBOXVMM_EXIT_SGDT_ENABLED_RAW()
10036 | VBOXVMM_EXIT_LGDT_ENABLED_RAW()
10037 | VBOXVMM_EXIT_SLDT_ENABLED_RAW()
10038 | VBOXVMM_EXIT_LLDT_ENABLED_RAW()
10039 | VBOXVMM_EXIT_STR_ENABLED_RAW()
10040 | VBOXVMM_EXIT_LTR_ENABLED_RAW()
10041 | VBOXVMM_EXIT_GETSEC_ENABLED_RAW()
10042 | VBOXVMM_EXIT_RSM_ENABLED_RAW()
10043 | VBOXVMM_EXIT_RDRAND_ENABLED_RAW()
10044 | VBOXVMM_EXIT_RDSEED_ENABLED_RAW()
10045 | VBOXVMM_EXIT_XSAVES_ENABLED_RAW()
10046 | VBOXVMM_EXIT_XRSTORS_ENABLED_RAW()
10047 | VBOXVMM_EXIT_VMM_CALL_ENABLED_RAW()
10048 | VBOXVMM_EXIT_VMX_VMCLEAR_ENABLED_RAW()
10049 | VBOXVMM_EXIT_VMX_VMLAUNCH_ENABLED_RAW()
10050 | VBOXVMM_EXIT_VMX_VMPTRLD_ENABLED_RAW()
10051 | VBOXVMM_EXIT_VMX_VMPTRST_ENABLED_RAW()
10052 | VBOXVMM_EXIT_VMX_VMREAD_ENABLED_RAW()
10053 | VBOXVMM_EXIT_VMX_VMRESUME_ENABLED_RAW()
10054 | VBOXVMM_EXIT_VMX_VMWRITE_ENABLED_RAW()
10055 | VBOXVMM_EXIT_VMX_VMXOFF_ENABLED_RAW()
10056 | VBOXVMM_EXIT_VMX_VMXON_ENABLED_RAW()
10057 | VBOXVMM_EXIT_VMX_VMFUNC_ENABLED_RAW()
10058 | VBOXVMM_EXIT_VMX_INVEPT_ENABLED_RAW()
10059 | VBOXVMM_EXIT_VMX_INVVPID_ENABLED_RAW()
10060 | VBOXVMM_EXIT_VMX_INVPCID_ENABLED_RAW()
10061 | VBOXVMM_EXIT_VMX_EPT_VIOLATION_ENABLED_RAW()
10062 | VBOXVMM_EXIT_VMX_EPT_MISCONFIG_ENABLED_RAW()
10063 | VBOXVMM_EXIT_VMX_VAPIC_ACCESS_ENABLED_RAW()
10064 | VBOXVMM_EXIT_VMX_VAPIC_WRITE_ENABLED_RAW()
10065 ) != 0;
10066}
10067
10068
10069/**
10070 * Runs the guest code using VT-x.
10071 *
10072 * @returns Strict VBox status code (i.e. informational status codes too).
10073 * @param pVCpu The cross context virtual CPU structure.
10074 * @param pCtx Pointer to the guest-CPU context.
10075 */
10076VMMR0DECL(VBOXSTRICTRC) VMXR0RunGuestCode(PVMCPU pVCpu, PCPUMCTX pCtx)
10077{
10078 Assert(VMMRZCallRing3IsEnabled(pVCpu));
10079 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10080 HMVMX_ASSERT_PREEMPT_SAFE();
10081
10082 VMMRZCallRing3SetNotification(pVCpu, hmR0VmxCallRing3Callback, pCtx);
10083
10084 VBOXSTRICTRC rcStrict;
10085 if ( !pVCpu->hm.s.fUseDebugLoop
10086 && (!VBOXVMM_ANY_PROBES_ENABLED() || !hmR0VmxAnyExpensiveProbesEnabled())
10087 && !DBGFIsStepping(pVCpu)
10088 && !pVCpu->CTX_SUFF(pVM)->dbgf.ro.cEnabledInt3Breakpoints)
10089 rcStrict = hmR0VmxRunGuestCodeNormal(pVCpu, pCtx);
10090 else
10091 rcStrict = hmR0VmxRunGuestCodeDebug(pVCpu, pCtx);
10092
10093 if (rcStrict == VERR_EM_INTERPRETER)
10094 rcStrict = VINF_EM_RAW_EMULATE_INSTR;
10095 else if (rcStrict == VINF_EM_RESET)
10096 rcStrict = VINF_EM_TRIPLE_FAULT;
10097
10098 int rc2 = hmR0VmxExitToRing3(pVCpu, pCtx, rcStrict);
10099 if (RT_FAILURE(rc2))
10100 {
10101 pVCpu->hm.s.u32HMError = (uint32_t)VBOXSTRICTRC_VAL(rcStrict);
10102 rcStrict = rc2;
10103 }
10104 Assert(!ASMAtomicUoReadU64(&pCtx->fExtrn));
10105 Assert(!VMMRZCallRing3IsNotificationSet(pVCpu));
10106 return rcStrict;
10107}
10108
10109
10110#ifndef HMVMX_USE_FUNCTION_TABLE
10111DECLINLINE(VBOXSTRICTRC) hmR0VmxHandleExit(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient, uint32_t rcReason)
10112{
10113#ifdef DEBUG_ramshankar
10114#define VMEXIT_CALL_RET(a_fSave, a_CallExpr) \
10115 do { \
10116 if (a_fSave != 0) \
10117 hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL); \
10118 VBOXSTRICTRC rcStrict = a_CallExpr; \
10119 if (a_fSave != 0) \
10120 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST); \
10121 return rcStrict; \
10122 } while (0)
10123#else
10124# define VMEXIT_CALL_RET(a_fSave, a_CallExpr) return a_CallExpr
10125#endif
10126 switch (rcReason)
10127 {
10128 case VMX_EXIT_EPT_MISCONFIG: VMEXIT_CALL_RET(0, hmR0VmxExitEptMisconfig(pVCpu, pMixedCtx, pVmxTransient));
10129 case VMX_EXIT_EPT_VIOLATION: VMEXIT_CALL_RET(0, hmR0VmxExitEptViolation(pVCpu, pMixedCtx, pVmxTransient));
10130 case VMX_EXIT_IO_INSTR: VMEXIT_CALL_RET(0, hmR0VmxExitIoInstr(pVCpu, pMixedCtx, pVmxTransient));
10131 case VMX_EXIT_CPUID: VMEXIT_CALL_RET(0, hmR0VmxExitCpuid(pVCpu, pMixedCtx, pVmxTransient));
10132 case VMX_EXIT_RDTSC: VMEXIT_CALL_RET(0, hmR0VmxExitRdtsc(pVCpu, pMixedCtx, pVmxTransient));
10133 case VMX_EXIT_RDTSCP: VMEXIT_CALL_RET(0, hmR0VmxExitRdtscp(pVCpu, pMixedCtx, pVmxTransient));
10134 case VMX_EXIT_APIC_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitApicAccess(pVCpu, pMixedCtx, pVmxTransient));
10135 case VMX_EXIT_XCPT_OR_NMI: VMEXIT_CALL_RET(0, hmR0VmxExitXcptOrNmi(pVCpu, pMixedCtx, pVmxTransient));
10136 case VMX_EXIT_MOV_CRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovCRx(pVCpu, pMixedCtx, pVmxTransient));
10137 case VMX_EXIT_EXT_INT: VMEXIT_CALL_RET(0, hmR0VmxExitExtInt(pVCpu, pMixedCtx, pVmxTransient));
10138 case VMX_EXIT_INT_WINDOW: VMEXIT_CALL_RET(0, hmR0VmxExitIntWindow(pVCpu, pMixedCtx, pVmxTransient));
10139 case VMX_EXIT_TPR_BELOW_THRESHOLD: VMEXIT_CALL_RET(0, hmR0VmxExitTprBelowThreshold(pVCpu, pMixedCtx, pVmxTransient));
10140 case VMX_EXIT_MWAIT: VMEXIT_CALL_RET(0, hmR0VmxExitMwait(pVCpu, pMixedCtx, pVmxTransient));
10141 case VMX_EXIT_MONITOR: VMEXIT_CALL_RET(0, hmR0VmxExitMonitor(pVCpu, pMixedCtx, pVmxTransient));
10142 case VMX_EXIT_TASK_SWITCH: VMEXIT_CALL_RET(0, hmR0VmxExitTaskSwitch(pVCpu, pMixedCtx, pVmxTransient));
10143 case VMX_EXIT_PREEMPT_TIMER: VMEXIT_CALL_RET(0, hmR0VmxExitPreemptTimer(pVCpu, pMixedCtx, pVmxTransient));
10144 case VMX_EXIT_RDMSR: VMEXIT_CALL_RET(0, hmR0VmxExitRdmsr(pVCpu, pMixedCtx, pVmxTransient));
10145 case VMX_EXIT_WRMSR: VMEXIT_CALL_RET(0, hmR0VmxExitWrmsr(pVCpu, pMixedCtx, pVmxTransient));
10146 case VMX_EXIT_VMCALL: VMEXIT_CALL_RET(0, hmR0VmxExitVmcall(pVCpu, pMixedCtx, pVmxTransient));
10147 case VMX_EXIT_MOV_DRX: VMEXIT_CALL_RET(0, hmR0VmxExitMovDRx(pVCpu, pMixedCtx, pVmxTransient));
10148 case VMX_EXIT_HLT: VMEXIT_CALL_RET(0, hmR0VmxExitHlt(pVCpu, pMixedCtx, pVmxTransient));
10149 case VMX_EXIT_INVD: VMEXIT_CALL_RET(0, hmR0VmxExitInvd(pVCpu, pMixedCtx, pVmxTransient));
10150 case VMX_EXIT_INVLPG: VMEXIT_CALL_RET(0, hmR0VmxExitInvlpg(pVCpu, pMixedCtx, pVmxTransient));
10151 case VMX_EXIT_RSM: VMEXIT_CALL_RET(0, hmR0VmxExitRsm(pVCpu, pMixedCtx, pVmxTransient));
10152 case VMX_EXIT_MTF: VMEXIT_CALL_RET(0, hmR0VmxExitMtf(pVCpu, pMixedCtx, pVmxTransient));
10153 case VMX_EXIT_PAUSE: VMEXIT_CALL_RET(0, hmR0VmxExitPause(pVCpu, pMixedCtx, pVmxTransient));
10154 case VMX_EXIT_XDTR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10155 case VMX_EXIT_TR_ACCESS: VMEXIT_CALL_RET(0, hmR0VmxExitXdtrAccess(pVCpu, pMixedCtx, pVmxTransient));
10156 case VMX_EXIT_WBINVD: VMEXIT_CALL_RET(0, hmR0VmxExitWbinvd(pVCpu, pMixedCtx, pVmxTransient));
10157 case VMX_EXIT_XSETBV: VMEXIT_CALL_RET(0, hmR0VmxExitXsetbv(pVCpu, pMixedCtx, pVmxTransient));
10158 case VMX_EXIT_RDRAND: VMEXIT_CALL_RET(0, hmR0VmxExitRdrand(pVCpu, pMixedCtx, pVmxTransient));
10159 case VMX_EXIT_INVPCID: VMEXIT_CALL_RET(0, hmR0VmxExitInvpcid(pVCpu, pMixedCtx, pVmxTransient));
10160 case VMX_EXIT_GETSEC: VMEXIT_CALL_RET(0, hmR0VmxExitGetsec(pVCpu, pMixedCtx, pVmxTransient));
10161 case VMX_EXIT_RDPMC: VMEXIT_CALL_RET(0, hmR0VmxExitRdpmc(pVCpu, pMixedCtx, pVmxTransient));
10162
10163 case VMX_EXIT_TRIPLE_FAULT: return hmR0VmxExitTripleFault(pVCpu, pMixedCtx, pVmxTransient);
10164 case VMX_EXIT_NMI_WINDOW: return hmR0VmxExitNmiWindow(pVCpu, pMixedCtx, pVmxTransient);
10165 case VMX_EXIT_INIT_SIGNAL: return hmR0VmxExitInitSignal(pVCpu, pMixedCtx, pVmxTransient);
10166 case VMX_EXIT_SIPI: return hmR0VmxExitSipi(pVCpu, pMixedCtx, pVmxTransient);
10167 case VMX_EXIT_IO_SMI: return hmR0VmxExitIoSmi(pVCpu, pMixedCtx, pVmxTransient);
10168 case VMX_EXIT_SMI: return hmR0VmxExitSmi(pVCpu, pMixedCtx, pVmxTransient);
10169 case VMX_EXIT_ERR_MSR_LOAD: return hmR0VmxExitErrMsrLoad(pVCpu, pMixedCtx, pVmxTransient);
10170 case VMX_EXIT_ERR_INVALID_GUEST_STATE: return hmR0VmxExitErrInvalidGuestState(pVCpu, pMixedCtx, pVmxTransient);
10171 case VMX_EXIT_ERR_MACHINE_CHECK: return hmR0VmxExitErrMachineCheck(pVCpu, pMixedCtx, pVmxTransient);
10172
10173 case VMX_EXIT_VMCLEAR:
10174 case VMX_EXIT_VMLAUNCH:
10175 case VMX_EXIT_VMPTRLD:
10176 case VMX_EXIT_VMPTRST:
10177 case VMX_EXIT_VMREAD:
10178 case VMX_EXIT_VMRESUME:
10179 case VMX_EXIT_VMWRITE:
10180 case VMX_EXIT_VMXOFF:
10181 case VMX_EXIT_VMXON:
10182 case VMX_EXIT_INVEPT:
10183 case VMX_EXIT_INVVPID:
10184 case VMX_EXIT_VMFUNC:
10185 case VMX_EXIT_XSAVES:
10186 case VMX_EXIT_XRSTORS:
10187 return hmR0VmxExitSetPendingXcptUD(pVCpu, pMixedCtx, pVmxTransient);
10188
10189 case VMX_EXIT_ENCLS:
10190 case VMX_EXIT_RDSEED: /* only spurious VM-exits, so undefined */
10191 case VMX_EXIT_PML_FULL:
10192 default:
10193 return hmR0VmxExitErrUndefined(pVCpu, pMixedCtx, pVmxTransient);
10194 }
10195#undef VMEXIT_CALL_RET
10196}
10197#endif /* !HMVMX_USE_FUNCTION_TABLE */
10198
10199
10200#ifdef VBOX_STRICT
10201/* Is there some generic IPRT define for this that are not in Runtime/internal/\* ?? */
10202# define HMVMX_ASSERT_PREEMPT_CPUID_VAR() \
10203 RTCPUID const idAssertCpu = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId()
10204
10205# define HMVMX_ASSERT_PREEMPT_CPUID() \
10206 do { \
10207 RTCPUID const idAssertCpuNow = RTThreadPreemptIsEnabled(NIL_RTTHREAD) ? NIL_RTCPUID : RTMpCpuId(); \
10208 AssertMsg(idAssertCpu == idAssertCpuNow, ("VMX %#x, %#x\n", idAssertCpu, idAssertCpuNow)); \
10209 } while (0)
10210
10211# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10212 do { \
10213 AssertPtr(pVCpu); \
10214 AssertPtr(pMixedCtx); \
10215 AssertPtr(pVmxTransient); \
10216 Assert(pVmxTransient->fVMEntryFailed == false); \
10217 Assert(ASMIntAreEnabled()); \
10218 HMVMX_ASSERT_PREEMPT_SAFE(); \
10219 HMVMX_ASSERT_PREEMPT_CPUID_VAR(); \
10220 Log4Func(("vcpu[%RU32] -v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v-v\n", pVCpu->idCpu)); \
10221 HMVMX_ASSERT_PREEMPT_SAFE(); \
10222 if (VMMR0IsLogFlushDisabled(pVCpu)) \
10223 HMVMX_ASSERT_PREEMPT_CPUID(); \
10224 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10225 } while (0)
10226
10227# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() \
10228 do { \
10229 Log4Func(("\n")); \
10230 } while (0)
10231#else /* nonstrict builds: */
10232# define HMVMX_VALIDATE_EXIT_HANDLER_PARAMS() \
10233 do { \
10234 HMVMX_STOP_EXIT_DISPATCH_PROF(); \
10235 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient); \
10236 } while (0)
10237# define HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS() do { } while (0)
10238#endif
10239
10240
10241/**
10242 * Advances the guest RIP by the specified number of bytes.
10243 *
10244 * @param pVCpu The cross context virtual CPU structure.
10245 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10246 * out-of-sync. Make sure to update the required fields
10247 * before using them.
10248 * @param cbInstr Number of bytes to advance the RIP by.
10249 *
10250 * @remarks No-long-jump zone!!!
10251 */
10252DECLINLINE(void) hmR0VmxAdvanceGuestRipBy(PVMCPU pVCpu, PCPUMCTX pMixedCtx, uint32_t cbInstr)
10253{
10254 /* Advance the RIP. */
10255 pMixedCtx->rip += cbInstr;
10256 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
10257
10258 /* Update interrupt inhibition. */
10259 if ( VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS)
10260 && pMixedCtx->rip != EMGetInhibitInterruptsPC(pVCpu))
10261 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
10262}
10263
10264
10265/**
10266 * Advances the guest RIP after reading it from the VMCS.
10267 *
10268 * @returns VBox status code, no informational status codes.
10269 * @param pVCpu The cross context virtual CPU structure.
10270 * @param pMixedCtx Pointer to the guest-CPU context. The data maybe
10271 * out-of-sync. Make sure to update the required fields
10272 * before using them.
10273 * @param pVmxTransient Pointer to the VMX transient structure.
10274 *
10275 * @remarks No-long-jump zone!!!
10276 */
10277static int hmR0VmxAdvanceGuestRip(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10278{
10279 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
10280 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
10281 | CPUMCTX_EXTRN_RFLAGS);
10282 AssertRCReturn(rc, rc);
10283
10284 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, pVmxTransient->cbInstr);
10285
10286 /*
10287 * Deliver a debug exception to the guest if it is single-stepping. Don't directly inject a #DB but use the
10288 * pending debug exception field as it takes care of priority of events.
10289 *
10290 * See Intel spec. 32.2.1 "Debug Exceptions".
10291 */
10292 if ( !pVCpu->hm.s.fSingleInstruction
10293 && pMixedCtx->eflags.Bits.u1TF)
10294 {
10295 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
10296 AssertRCReturn(rc, rc);
10297 }
10298
10299 return VINF_SUCCESS;
10300}
10301
10302
10303/**
10304 * Tries to determine what part of the guest-state VT-x has deemed as invalid
10305 * and update error record fields accordingly.
10306 *
10307 * @return VMX_IGS_* return codes.
10308 * @retval VMX_IGS_REASON_NOT_FOUND if this function could not find anything
10309 * wrong with the guest state.
10310 *
10311 * @param pVCpu The cross context virtual CPU structure.
10312 * @param pCtx Pointer to the guest-CPU state.
10313 *
10314 * @remarks This function assumes our cache of the VMCS controls
10315 * are valid, i.e. hmR0VmxCheckVmcsCtls() succeeded.
10316 */
10317static uint32_t hmR0VmxCheckGuestState(PVMCPU pVCpu, PCPUMCTX pCtx)
10318{
10319#define HMVMX_ERROR_BREAK(err) { uError = (err); break; }
10320#define HMVMX_CHECK_BREAK(expr, err) if (!(expr)) { \
10321 uError = (err); \
10322 break; \
10323 } else do { } while (0)
10324
10325 int rc;
10326 PVM pVM = pVCpu->CTX_SUFF(pVM);
10327 uint32_t uError = VMX_IGS_ERROR;
10328 uint32_t u32Val;
10329 bool const fUnrestrictedGuest = pVM->hm.s.vmx.fUnrestrictedGuest;
10330
10331 do
10332 {
10333 /*
10334 * CR0.
10335 */
10336 uint32_t fSetCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10337 uint32_t const fZapCR0 = (uint32_t)(pVM->hm.s.vmx.Msrs.u64Cr0Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr0Fixed1);
10338 /* Exceptions for unrestricted-guests for fixed CR0 bits (PE, PG).
10339 See Intel spec. 26.3.1 "Checks on Guest Control Registers, Debug Registers and MSRs." */
10340 if (fUnrestrictedGuest)
10341 fSetCR0 &= ~(X86_CR0_PE | X86_CR0_PG);
10342
10343 uint32_t uGuestCR0;
10344 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &uGuestCR0);
10345 AssertRCBreak(rc);
10346 HMVMX_CHECK_BREAK((uGuestCR0 & fSetCR0) == fSetCR0, VMX_IGS_CR0_FIXED1);
10347 HMVMX_CHECK_BREAK(!(uGuestCR0 & ~fZapCR0), VMX_IGS_CR0_FIXED0);
10348 if ( !fUnrestrictedGuest
10349 && (uGuestCR0 & X86_CR0_PG)
10350 && !(uGuestCR0 & X86_CR0_PE))
10351 {
10352 HMVMX_ERROR_BREAK(VMX_IGS_CR0_PG_PE_COMBO);
10353 }
10354
10355 /*
10356 * CR4.
10357 */
10358 uint64_t const fSetCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 & pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10359 uint64_t const fZapCR4 = (pVM->hm.s.vmx.Msrs.u64Cr4Fixed0 | pVM->hm.s.vmx.Msrs.u64Cr4Fixed1);
10360
10361 uint32_t uGuestCR4;
10362 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR4, &uGuestCR4);
10363 AssertRCBreak(rc);
10364 HMVMX_CHECK_BREAK((uGuestCR4 & fSetCR4) == fSetCR4, VMX_IGS_CR4_FIXED1);
10365 HMVMX_CHECK_BREAK(!(uGuestCR4 & ~fZapCR4), VMX_IGS_CR4_FIXED0);
10366
10367 /*
10368 * IA32_DEBUGCTL MSR.
10369 */
10370 uint64_t u64Val;
10371 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_DEBUGCTL_FULL, &u64Val);
10372 AssertRCBreak(rc);
10373 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10374 && (u64Val & 0xfffffe3c)) /* Bits 31:9, bits 5:2 MBZ. */
10375 {
10376 HMVMX_ERROR_BREAK(VMX_IGS_DEBUGCTL_MSR_RESERVED);
10377 }
10378 uint64_t u64DebugCtlMsr = u64Val;
10379
10380#ifdef VBOX_STRICT
10381 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY, &u32Val);
10382 AssertRCBreak(rc);
10383 Assert(u32Val == pVCpu->hm.s.vmx.u32EntryCtls);
10384#endif
10385 bool const fLongModeGuest = RT_BOOL(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST);
10386
10387 /*
10388 * RIP and RFLAGS.
10389 */
10390 uint32_t u32Eflags;
10391#if HC_ARCH_BITS == 64
10392 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RIP, &u64Val);
10393 AssertRCBreak(rc);
10394 /* pCtx->rip can be different than the one in the VMCS (e.g. run guest code and VM-exits that don't update it). */
10395 if ( !fLongModeGuest
10396 || !pCtx->cs.Attr.n.u1Long)
10397 {
10398 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffff00000000)), VMX_IGS_LONGMODE_RIP_INVALID);
10399 }
10400 /** @todo If the processor supports N < 64 linear-address bits, bits 63:N
10401 * must be identical if the "IA-32e mode guest" VM-entry
10402 * control is 1 and CS.L is 1. No check applies if the
10403 * CPU supports 64 linear-address bits. */
10404
10405 /* Flags in pCtx can be different (real-on-v86 for instance). We are only concerned about the VMCS contents here. */
10406 rc = VMXReadVmcs64(VMX_VMCS_GUEST_RFLAGS, &u64Val);
10407 AssertRCBreak(rc);
10408 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffc08028)), /* Bit 63:22, Bit 15, 5, 3 MBZ. */
10409 VMX_IGS_RFLAGS_RESERVED);
10410 HMVMX_CHECK_BREAK((u64Val & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10411 u32Eflags = u64Val;
10412#else
10413 rc = VMXReadVmcs32(VMX_VMCS_GUEST_RFLAGS, &u32Eflags);
10414 AssertRCBreak(rc);
10415 HMVMX_CHECK_BREAK(!(u32Eflags & 0xffc08028), VMX_IGS_RFLAGS_RESERVED); /* Bit 31:22, Bit 15, 5, 3 MBZ. */
10416 HMVMX_CHECK_BREAK((u32Eflags & X86_EFL_RA1_MASK), VMX_IGS_RFLAGS_RESERVED1); /* Bit 1 MB1. */
10417#endif
10418
10419 if ( fLongModeGuest
10420 || ( fUnrestrictedGuest
10421 && !(uGuestCR0 & X86_CR0_PE)))
10422 {
10423 HMVMX_CHECK_BREAK(!(u32Eflags & X86_EFL_VM), VMX_IGS_RFLAGS_VM_INVALID);
10424 }
10425
10426 uint32_t u32EntryInfo;
10427 rc = VMXReadVmcs32(VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO, &u32EntryInfo);
10428 AssertRCBreak(rc);
10429 if ( VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10430 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10431 {
10432 HMVMX_CHECK_BREAK(u32Eflags & X86_EFL_IF, VMX_IGS_RFLAGS_IF_INVALID);
10433 }
10434
10435 /*
10436 * 64-bit checks.
10437 */
10438#if HC_ARCH_BITS == 64
10439 if (fLongModeGuest)
10440 {
10441 HMVMX_CHECK_BREAK(uGuestCR0 & X86_CR0_PG, VMX_IGS_CR0_PG_LONGMODE);
10442 HMVMX_CHECK_BREAK(uGuestCR4 & X86_CR4_PAE, VMX_IGS_CR4_PAE_LONGMODE);
10443 }
10444
10445 if ( !fLongModeGuest
10446 && (uGuestCR4 & X86_CR4_PCIDE))
10447 {
10448 HMVMX_ERROR_BREAK(VMX_IGS_CR4_PCIDE);
10449 }
10450
10451 /** @todo CR3 field must be such that bits 63:52 and bits in the range
10452 * 51:32 beyond the processor's physical-address width are 0. */
10453
10454 if ( (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_DEBUG)
10455 && (pCtx->dr[7] & X86_DR7_MBZ_MASK))
10456 {
10457 HMVMX_ERROR_BREAK(VMX_IGS_DR7_RESERVED);
10458 }
10459
10460 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_ESP, &u64Val);
10461 AssertRCBreak(rc);
10462 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_ESP_NOT_CANONICAL);
10463
10464 rc = VMXReadVmcs64(VMX_VMCS_HOST_SYSENTER_EIP, &u64Val);
10465 AssertRCBreak(rc);
10466 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_SYSENTER_EIP_NOT_CANONICAL);
10467#endif
10468
10469 /*
10470 * PERF_GLOBAL MSR.
10471 */
10472 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PERF_MSR)
10473 {
10474 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PERF_GLOBAL_CTRL_FULL, &u64Val);
10475 AssertRCBreak(rc);
10476 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffff8fffffffc)),
10477 VMX_IGS_PERF_GLOBAL_MSR_RESERVED); /* Bits 63:35, bits 31:2 MBZ. */
10478 }
10479
10480 /*
10481 * PAT MSR.
10482 */
10483 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_PAT_MSR)
10484 {
10485 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PAT_FULL, &u64Val);
10486 AssertRCBreak(rc);
10487 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0x707070707070707)), VMX_IGS_PAT_MSR_RESERVED);
10488 for (unsigned i = 0; i < 8; i++)
10489 {
10490 uint8_t u8Val = (u64Val & 0xff);
10491 if ( u8Val != 0 /* UC */
10492 && u8Val != 1 /* WC */
10493 && u8Val != 4 /* WT */
10494 && u8Val != 5 /* WP */
10495 && u8Val != 6 /* WB */
10496 && u8Val != 7 /* UC- */)
10497 {
10498 HMVMX_ERROR_BREAK(VMX_IGS_PAT_MSR_INVALID);
10499 }
10500 u64Val >>= 8;
10501 }
10502 }
10503
10504 /*
10505 * EFER MSR.
10506 */
10507 if (pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_LOAD_GUEST_EFER_MSR)
10508 {
10509 Assert(pVM->hm.s.vmx.fSupportsVmcsEfer);
10510 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_EFER_FULL, &u64Val);
10511 AssertRCBreak(rc);
10512 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xfffffffffffff2fe)),
10513 VMX_IGS_EFER_MSR_RESERVED); /* Bits 63:12, bit 9, bits 7:1 MBZ. */
10514 HMVMX_CHECK_BREAK(RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL( pVCpu->hm.s.vmx.u32EntryCtls
10515 & VMX_VMCS_CTRL_ENTRY_IA32E_MODE_GUEST),
10516 VMX_IGS_EFER_LMA_GUEST_MODE_MISMATCH);
10517 HMVMX_CHECK_BREAK( fUnrestrictedGuest
10518 || !(uGuestCR0 & X86_CR0_PG)
10519 || RT_BOOL(u64Val & MSR_K6_EFER_LMA) == RT_BOOL(u64Val & MSR_K6_EFER_LME),
10520 VMX_IGS_EFER_LMA_LME_MISMATCH);
10521 }
10522
10523 /*
10524 * Segment registers.
10525 */
10526 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10527 || !(pCtx->ldtr.Sel & X86_SEL_LDT), VMX_IGS_LDTR_TI_INVALID);
10528 if (!(u32Eflags & X86_EFL_VM))
10529 {
10530 /* CS */
10531 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1Present, VMX_IGS_CS_ATTR_P_INVALID);
10532 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xf00), VMX_IGS_CS_ATTR_RESERVED);
10533 HMVMX_CHECK_BREAK(!(pCtx->cs.Attr.u & 0xfffe0000), VMX_IGS_CS_ATTR_RESERVED);
10534 HMVMX_CHECK_BREAK( (pCtx->cs.u32Limit & 0xfff) == 0xfff
10535 || !(pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10536 HMVMX_CHECK_BREAK( !(pCtx->cs.u32Limit & 0xfff00000)
10537 || (pCtx->cs.Attr.n.u1Granularity), VMX_IGS_CS_ATTR_G_INVALID);
10538 /* CS cannot be loaded with NULL in protected mode. */
10539 HMVMX_CHECK_BREAK(pCtx->cs.Attr.u && !(pCtx->cs.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_CS_ATTR_UNUSABLE);
10540 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u1DescType, VMX_IGS_CS_ATTR_S_INVALID);
10541 if (pCtx->cs.Attr.n.u4Type == 9 || pCtx->cs.Attr.n.u4Type == 11)
10542 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_UNEQUAL);
10543 else if (pCtx->cs.Attr.n.u4Type == 13 || pCtx->cs.Attr.n.u4Type == 15)
10544 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl <= pCtx->ss.Attr.n.u2Dpl, VMX_IGS_CS_SS_ATTR_DPL_MISMATCH);
10545 else if (pVM->hm.s.vmx.fUnrestrictedGuest && pCtx->cs.Attr.n.u4Type == 3)
10546 HMVMX_CHECK_BREAK(pCtx->cs.Attr.n.u2Dpl == 0, VMX_IGS_CS_ATTR_DPL_INVALID);
10547 else
10548 HMVMX_ERROR_BREAK(VMX_IGS_CS_ATTR_TYPE_INVALID);
10549
10550 /* SS */
10551 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10552 || (pCtx->ss.Sel & X86_SEL_RPL) == (pCtx->cs.Sel & X86_SEL_RPL), VMX_IGS_SS_CS_RPL_UNEQUAL);
10553 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u2Dpl == (pCtx->ss.Sel & X86_SEL_RPL), VMX_IGS_SS_ATTR_DPL_RPL_UNEQUAL);
10554 if ( !(pCtx->cr0 & X86_CR0_PE)
10555 || pCtx->cs.Attr.n.u4Type == 3)
10556 {
10557 HMVMX_CHECK_BREAK(!pCtx->ss.Attr.n.u2Dpl, VMX_IGS_SS_ATTR_DPL_INVALID);
10558 }
10559 if (!(pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE))
10560 {
10561 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u4Type == 3 || pCtx->ss.Attr.n.u4Type == 7, VMX_IGS_SS_ATTR_TYPE_INVALID);
10562 HMVMX_CHECK_BREAK(pCtx->ss.Attr.n.u1Present, VMX_IGS_SS_ATTR_P_INVALID);
10563 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xf00), VMX_IGS_SS_ATTR_RESERVED);
10564 HMVMX_CHECK_BREAK(!(pCtx->ss.Attr.u & 0xfffe0000), VMX_IGS_SS_ATTR_RESERVED);
10565 HMVMX_CHECK_BREAK( (pCtx->ss.u32Limit & 0xfff) == 0xfff
10566 || !(pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10567 HMVMX_CHECK_BREAK( !(pCtx->ss.u32Limit & 0xfff00000)
10568 || (pCtx->ss.Attr.n.u1Granularity), VMX_IGS_SS_ATTR_G_INVALID);
10569 }
10570
10571 /* DS, ES, FS, GS - only check for usable selectors, see hmR0VmxWriteSegmentReg(). */
10572 if (!(pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE))
10573 {
10574 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_DS_ATTR_A_INVALID);
10575 HMVMX_CHECK_BREAK(pCtx->ds.Attr.n.u1Present, VMX_IGS_DS_ATTR_P_INVALID);
10576 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10577 || pCtx->ds.Attr.n.u4Type > 11
10578 || pCtx->ds.Attr.n.u2Dpl >= (pCtx->ds.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10579 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xf00), VMX_IGS_DS_ATTR_RESERVED);
10580 HMVMX_CHECK_BREAK(!(pCtx->ds.Attr.u & 0xfffe0000), VMX_IGS_DS_ATTR_RESERVED);
10581 HMVMX_CHECK_BREAK( (pCtx->ds.u32Limit & 0xfff) == 0xfff
10582 || !(pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10583 HMVMX_CHECK_BREAK( !(pCtx->ds.u32Limit & 0xfff00000)
10584 || (pCtx->ds.Attr.n.u1Granularity), VMX_IGS_DS_ATTR_G_INVALID);
10585 HMVMX_CHECK_BREAK( !(pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10586 || (pCtx->ds.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_DS_ATTR_TYPE_INVALID);
10587 }
10588 if (!(pCtx->es.Attr.u & X86DESCATTR_UNUSABLE))
10589 {
10590 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_ES_ATTR_A_INVALID);
10591 HMVMX_CHECK_BREAK(pCtx->es.Attr.n.u1Present, VMX_IGS_ES_ATTR_P_INVALID);
10592 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10593 || pCtx->es.Attr.n.u4Type > 11
10594 || pCtx->es.Attr.n.u2Dpl >= (pCtx->es.Sel & X86_SEL_RPL), VMX_IGS_DS_ATTR_DPL_RPL_UNEQUAL);
10595 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xf00), VMX_IGS_ES_ATTR_RESERVED);
10596 HMVMX_CHECK_BREAK(!(pCtx->es.Attr.u & 0xfffe0000), VMX_IGS_ES_ATTR_RESERVED);
10597 HMVMX_CHECK_BREAK( (pCtx->es.u32Limit & 0xfff) == 0xfff
10598 || !(pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10599 HMVMX_CHECK_BREAK( !(pCtx->es.u32Limit & 0xfff00000)
10600 || (pCtx->es.Attr.n.u1Granularity), VMX_IGS_ES_ATTR_G_INVALID);
10601 HMVMX_CHECK_BREAK( !(pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10602 || (pCtx->es.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_ES_ATTR_TYPE_INVALID);
10603 }
10604 if (!(pCtx->fs.Attr.u & X86DESCATTR_UNUSABLE))
10605 {
10606 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_FS_ATTR_A_INVALID);
10607 HMVMX_CHECK_BREAK(pCtx->fs.Attr.n.u1Present, VMX_IGS_FS_ATTR_P_INVALID);
10608 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10609 || pCtx->fs.Attr.n.u4Type > 11
10610 || pCtx->fs.Attr.n.u2Dpl >= (pCtx->fs.Sel & X86_SEL_RPL), VMX_IGS_FS_ATTR_DPL_RPL_UNEQUAL);
10611 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xf00), VMX_IGS_FS_ATTR_RESERVED);
10612 HMVMX_CHECK_BREAK(!(pCtx->fs.Attr.u & 0xfffe0000), VMX_IGS_FS_ATTR_RESERVED);
10613 HMVMX_CHECK_BREAK( (pCtx->fs.u32Limit & 0xfff) == 0xfff
10614 || !(pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10615 HMVMX_CHECK_BREAK( !(pCtx->fs.u32Limit & 0xfff00000)
10616 || (pCtx->fs.Attr.n.u1Granularity), VMX_IGS_FS_ATTR_G_INVALID);
10617 HMVMX_CHECK_BREAK( !(pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10618 || (pCtx->fs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_FS_ATTR_TYPE_INVALID);
10619 }
10620 if (!(pCtx->gs.Attr.u & X86DESCATTR_UNUSABLE))
10621 {
10622 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_ACCESSED, VMX_IGS_GS_ATTR_A_INVALID);
10623 HMVMX_CHECK_BREAK(pCtx->gs.Attr.n.u1Present, VMX_IGS_GS_ATTR_P_INVALID);
10624 HMVMX_CHECK_BREAK( pVM->hm.s.vmx.fUnrestrictedGuest
10625 || pCtx->gs.Attr.n.u4Type > 11
10626 || pCtx->gs.Attr.n.u2Dpl >= (pCtx->gs.Sel & X86_SEL_RPL), VMX_IGS_GS_ATTR_DPL_RPL_UNEQUAL);
10627 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xf00), VMX_IGS_GS_ATTR_RESERVED);
10628 HMVMX_CHECK_BREAK(!(pCtx->gs.Attr.u & 0xfffe0000), VMX_IGS_GS_ATTR_RESERVED);
10629 HMVMX_CHECK_BREAK( (pCtx->gs.u32Limit & 0xfff) == 0xfff
10630 || !(pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10631 HMVMX_CHECK_BREAK( !(pCtx->gs.u32Limit & 0xfff00000)
10632 || (pCtx->gs.Attr.n.u1Granularity), VMX_IGS_GS_ATTR_G_INVALID);
10633 HMVMX_CHECK_BREAK( !(pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_CODE)
10634 || (pCtx->gs.Attr.n.u4Type & X86_SEL_TYPE_READ), VMX_IGS_GS_ATTR_TYPE_INVALID);
10635 }
10636 /* 64-bit capable CPUs. */
10637#if HC_ARCH_BITS == 64
10638 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10639 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10640 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10641 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10642 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10643 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10644 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10645 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10646 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10647 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10648 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10649#endif
10650 }
10651 else
10652 {
10653 /* V86 mode checks. */
10654 uint32_t u32CSAttr, u32SSAttr, u32DSAttr, u32ESAttr, u32FSAttr, u32GSAttr;
10655 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
10656 {
10657 u32CSAttr = 0xf3; u32SSAttr = 0xf3;
10658 u32DSAttr = 0xf3; u32ESAttr = 0xf3;
10659 u32FSAttr = 0xf3; u32GSAttr = 0xf3;
10660 }
10661 else
10662 {
10663 u32CSAttr = pCtx->cs.Attr.u; u32SSAttr = pCtx->ss.Attr.u;
10664 u32DSAttr = pCtx->ds.Attr.u; u32ESAttr = pCtx->es.Attr.u;
10665 u32FSAttr = pCtx->fs.Attr.u; u32GSAttr = pCtx->gs.Attr.u;
10666 }
10667
10668 /* CS */
10669 HMVMX_CHECK_BREAK((pCtx->cs.u64Base == (uint64_t)pCtx->cs.Sel << 4), VMX_IGS_V86_CS_BASE_INVALID);
10670 HMVMX_CHECK_BREAK(pCtx->cs.u32Limit == 0xffff, VMX_IGS_V86_CS_LIMIT_INVALID);
10671 HMVMX_CHECK_BREAK(u32CSAttr == 0xf3, VMX_IGS_V86_CS_ATTR_INVALID);
10672 /* SS */
10673 HMVMX_CHECK_BREAK((pCtx->ss.u64Base == (uint64_t)pCtx->ss.Sel << 4), VMX_IGS_V86_SS_BASE_INVALID);
10674 HMVMX_CHECK_BREAK(pCtx->ss.u32Limit == 0xffff, VMX_IGS_V86_SS_LIMIT_INVALID);
10675 HMVMX_CHECK_BREAK(u32SSAttr == 0xf3, VMX_IGS_V86_SS_ATTR_INVALID);
10676 /* DS */
10677 HMVMX_CHECK_BREAK((pCtx->ds.u64Base == (uint64_t)pCtx->ds.Sel << 4), VMX_IGS_V86_DS_BASE_INVALID);
10678 HMVMX_CHECK_BREAK(pCtx->ds.u32Limit == 0xffff, VMX_IGS_V86_DS_LIMIT_INVALID);
10679 HMVMX_CHECK_BREAK(u32DSAttr == 0xf3, VMX_IGS_V86_DS_ATTR_INVALID);
10680 /* ES */
10681 HMVMX_CHECK_BREAK((pCtx->es.u64Base == (uint64_t)pCtx->es.Sel << 4), VMX_IGS_V86_ES_BASE_INVALID);
10682 HMVMX_CHECK_BREAK(pCtx->es.u32Limit == 0xffff, VMX_IGS_V86_ES_LIMIT_INVALID);
10683 HMVMX_CHECK_BREAK(u32ESAttr == 0xf3, VMX_IGS_V86_ES_ATTR_INVALID);
10684 /* FS */
10685 HMVMX_CHECK_BREAK((pCtx->fs.u64Base == (uint64_t)pCtx->fs.Sel << 4), VMX_IGS_V86_FS_BASE_INVALID);
10686 HMVMX_CHECK_BREAK(pCtx->fs.u32Limit == 0xffff, VMX_IGS_V86_FS_LIMIT_INVALID);
10687 HMVMX_CHECK_BREAK(u32FSAttr == 0xf3, VMX_IGS_V86_FS_ATTR_INVALID);
10688 /* GS */
10689 HMVMX_CHECK_BREAK((pCtx->gs.u64Base == (uint64_t)pCtx->gs.Sel << 4), VMX_IGS_V86_GS_BASE_INVALID);
10690 HMVMX_CHECK_BREAK(pCtx->gs.u32Limit == 0xffff, VMX_IGS_V86_GS_LIMIT_INVALID);
10691 HMVMX_CHECK_BREAK(u32GSAttr == 0xf3, VMX_IGS_V86_GS_ATTR_INVALID);
10692 /* 64-bit capable CPUs. */
10693#if HC_ARCH_BITS == 64
10694 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->fs.u64Base), VMX_IGS_FS_BASE_NOT_CANONICAL);
10695 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->gs.u64Base), VMX_IGS_GS_BASE_NOT_CANONICAL);
10696 HMVMX_CHECK_BREAK( (pCtx->ldtr.Attr.u & X86DESCATTR_UNUSABLE)
10697 || X86_IS_CANONICAL(pCtx->ldtr.u64Base), VMX_IGS_LDTR_BASE_NOT_CANONICAL);
10698 HMVMX_CHECK_BREAK(!RT_HI_U32(pCtx->cs.u64Base), VMX_IGS_LONGMODE_CS_BASE_INVALID);
10699 HMVMX_CHECK_BREAK((pCtx->ss.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ss.u64Base),
10700 VMX_IGS_LONGMODE_SS_BASE_INVALID);
10701 HMVMX_CHECK_BREAK((pCtx->ds.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->ds.u64Base),
10702 VMX_IGS_LONGMODE_DS_BASE_INVALID);
10703 HMVMX_CHECK_BREAK((pCtx->es.Attr.u & X86DESCATTR_UNUSABLE) || !RT_HI_U32(pCtx->es.u64Base),
10704 VMX_IGS_LONGMODE_ES_BASE_INVALID);
10705#endif
10706 }
10707
10708 /*
10709 * TR.
10710 */
10711 HMVMX_CHECK_BREAK(!(pCtx->tr.Sel & X86_SEL_LDT), VMX_IGS_TR_TI_INVALID);
10712 /* 64-bit capable CPUs. */
10713#if HC_ARCH_BITS == 64
10714 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(pCtx->tr.u64Base), VMX_IGS_TR_BASE_NOT_CANONICAL);
10715#endif
10716 if (fLongModeGuest)
10717 {
10718 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u4Type == 11, /* 64-bit busy TSS. */
10719 VMX_IGS_LONGMODE_TR_ATTR_TYPE_INVALID);
10720 }
10721 else
10722 {
10723 HMVMX_CHECK_BREAK( pCtx->tr.Attr.n.u4Type == 3 /* 16-bit busy TSS. */
10724 || pCtx->tr.Attr.n.u4Type == 11, /* 32-bit busy TSS.*/
10725 VMX_IGS_TR_ATTR_TYPE_INVALID);
10726 }
10727 HMVMX_CHECK_BREAK(!pCtx->tr.Attr.n.u1DescType, VMX_IGS_TR_ATTR_S_INVALID);
10728 HMVMX_CHECK_BREAK(pCtx->tr.Attr.n.u1Present, VMX_IGS_TR_ATTR_P_INVALID);
10729 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & 0xf00), VMX_IGS_TR_ATTR_RESERVED); /* Bits 11:8 MBZ. */
10730 HMVMX_CHECK_BREAK( (pCtx->tr.u32Limit & 0xfff) == 0xfff
10731 || !(pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10732 HMVMX_CHECK_BREAK( !(pCtx->tr.u32Limit & 0xfff00000)
10733 || (pCtx->tr.Attr.n.u1Granularity), VMX_IGS_TR_ATTR_G_INVALID);
10734 HMVMX_CHECK_BREAK(!(pCtx->tr.Attr.u & X86DESCATTR_UNUSABLE), VMX_IGS_TR_ATTR_UNUSABLE);
10735
10736 /*
10737 * GDTR and IDTR.
10738 */
10739#if HC_ARCH_BITS == 64
10740 rc = VMXReadVmcs64(VMX_VMCS_GUEST_GDTR_BASE, &u64Val);
10741 AssertRCBreak(rc);
10742 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_GDTR_BASE_NOT_CANONICAL);
10743
10744 rc = VMXReadVmcs64(VMX_VMCS_GUEST_IDTR_BASE, &u64Val);
10745 AssertRCBreak(rc);
10746 HMVMX_CHECK_BREAK(X86_IS_CANONICAL(u64Val), VMX_IGS_IDTR_BASE_NOT_CANONICAL);
10747#endif
10748
10749 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_GDTR_LIMIT, &u32Val);
10750 AssertRCBreak(rc);
10751 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_GDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10752
10753 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_IDTR_LIMIT, &u32Val);
10754 AssertRCBreak(rc);
10755 HMVMX_CHECK_BREAK(!(u32Val & 0xffff0000), VMX_IGS_IDTR_LIMIT_INVALID); /* Bits 31:16 MBZ. */
10756
10757 /*
10758 * Guest Non-Register State.
10759 */
10760 /* Activity State. */
10761 uint32_t u32ActivityState;
10762 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_ACTIVITY_STATE, &u32ActivityState);
10763 AssertRCBreak(rc);
10764 HMVMX_CHECK_BREAK( !u32ActivityState
10765 || (u32ActivityState & MSR_IA32_VMX_MISC_ACTIVITY_STATES(pVM->hm.s.vmx.Msrs.u64Misc)),
10766 VMX_IGS_ACTIVITY_STATE_INVALID);
10767 HMVMX_CHECK_BREAK( !(pCtx->ss.Attr.n.u2Dpl)
10768 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_HLT, VMX_IGS_ACTIVITY_STATE_HLT_INVALID);
10769 uint32_t u32IntrState;
10770 rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &u32IntrState);
10771 AssertRCBreak(rc);
10772 if ( u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS
10773 || u32IntrState == VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10774 {
10775 HMVMX_CHECK_BREAK(u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_ACTIVE, VMX_IGS_ACTIVITY_STATE_ACTIVE_INVALID);
10776 }
10777
10778 /** @todo Activity state and injecting interrupts. Left as a todo since we
10779 * currently don't use activity states but ACTIVE. */
10780
10781 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10782 || u32ActivityState != VMX_VMCS_GUEST_ACTIVITY_SIPI_WAIT, VMX_IGS_ACTIVITY_STATE_SIPI_WAIT_INVALID);
10783
10784 /* Guest interruptibility-state. */
10785 HMVMX_CHECK_BREAK(!(u32IntrState & 0xfffffff0), VMX_IGS_INTERRUPTIBILITY_STATE_RESERVED);
10786 HMVMX_CHECK_BREAK((u32IntrState & ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10787 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS))
10788 != ( VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI
10789 | VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10790 VMX_IGS_INTERRUPTIBILITY_STATE_STI_MOVSS_INVALID);
10791 HMVMX_CHECK_BREAK( (u32Eflags & X86_EFL_IF)
10792 || !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10793 VMX_IGS_INTERRUPTIBILITY_STATE_STI_EFL_INVALID);
10794 if (VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo))
10795 {
10796 if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT)
10797 {
10798 HMVMX_CHECK_BREAK( !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10799 && !(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10800 VMX_IGS_INTERRUPTIBILITY_STATE_EXT_INT_INVALID);
10801 }
10802 else if (VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10803 {
10804 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS),
10805 VMX_IGS_INTERRUPTIBILITY_STATE_MOVSS_INVALID);
10806 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI),
10807 VMX_IGS_INTERRUPTIBILITY_STATE_STI_INVALID);
10808 }
10809 }
10810 /** @todo Assumes the processor is not in SMM. */
10811 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10812 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_INVALID);
10813 HMVMX_CHECK_BREAK( !(pVCpu->hm.s.vmx.u32EntryCtls & VMX_VMCS_CTRL_ENTRY_ENTRY_SMM)
10814 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_SMI),
10815 VMX_IGS_INTERRUPTIBILITY_STATE_SMI_SMM_INVALID);
10816 if ( (pVCpu->hm.s.vmx.u32PinCtls & VMX_VMCS_CTRL_PIN_EXEC_VIRTUAL_NMI)
10817 && VMX_ENTRY_INTERRUPTION_INFO_IS_VALID(u32EntryInfo)
10818 && VMX_ENTRY_INTERRUPTION_INFO_TYPE(u32EntryInfo) == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10819 {
10820 HMVMX_CHECK_BREAK(!(u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_NMI),
10821 VMX_IGS_INTERRUPTIBILITY_STATE_NMI_INVALID);
10822 }
10823
10824 /* Pending debug exceptions. */
10825#if HC_ARCH_BITS == 64
10826 rc = VMXReadVmcs64(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u64Val);
10827 AssertRCBreak(rc);
10828 /* Bits 63:15, Bit 13, Bits 11:4 MBZ. */
10829 HMVMX_CHECK_BREAK(!(u64Val & UINT64_C(0xffffffffffffaff0)), VMX_IGS_LONGMODE_PENDING_DEBUG_RESERVED);
10830 u32Val = u64Val; /* For pending debug exceptions checks below. */
10831#else
10832 rc = VMXReadVmcs32(VMX_VMCS_GUEST_PENDING_DEBUG_EXCEPTIONS, &u32Val);
10833 AssertRCBreak(rc);
10834 /* Bits 31:15, Bit 13, Bits 11:4 MBZ. */
10835 HMVMX_CHECK_BREAK(!(u32Val & 0xffffaff0), VMX_IGS_PENDING_DEBUG_RESERVED);
10836#endif
10837
10838 if ( (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI)
10839 || (u32IntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_MOVSS)
10840 || u32ActivityState == VMX_VMCS_GUEST_ACTIVITY_HLT)
10841 {
10842 if ( (u32Eflags & X86_EFL_TF)
10843 && !(u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10844 {
10845 /* Bit 14 is PendingDebug.BS. */
10846 HMVMX_CHECK_BREAK(u32Val & RT_BIT(14), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_SET);
10847 }
10848 if ( !(u32Eflags & X86_EFL_TF)
10849 || (u64DebugCtlMsr & RT_BIT_64(1))) /* Bit 1 is IA32_DEBUGCTL.BTF. */
10850 {
10851 /* Bit 14 is PendingDebug.BS. */
10852 HMVMX_CHECK_BREAK(!(u32Val & RT_BIT(14)), VMX_IGS_PENDING_DEBUG_XCPT_BS_NOT_CLEAR);
10853 }
10854 }
10855
10856 /* VMCS link pointer. */
10857 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_VMCS_LINK_PTR_FULL, &u64Val);
10858 AssertRCBreak(rc);
10859 if (u64Val != UINT64_C(0xffffffffffffffff))
10860 {
10861 HMVMX_CHECK_BREAK(!(u64Val & 0xfff), VMX_IGS_VMCS_LINK_PTR_RESERVED);
10862 /** @todo Bits beyond the processor's physical-address width MBZ. */
10863 /** @todo 32-bit located in memory referenced by value of this field (as a
10864 * physical address) must contain the processor's VMCS revision ID. */
10865 /** @todo SMM checks. */
10866 }
10867
10868 /** @todo Checks on Guest Page-Directory-Pointer-Table Entries when guest is
10869 * not using Nested Paging? */
10870 if ( pVM->hm.s.fNestedPaging
10871 && !fLongModeGuest
10872 && CPUMIsGuestInPAEModeEx(pCtx))
10873 {
10874 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE0_FULL, &u64Val);
10875 AssertRCBreak(rc);
10876 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10877
10878 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE1_FULL, &u64Val);
10879 AssertRCBreak(rc);
10880 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10881
10882 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE2_FULL, &u64Val);
10883 AssertRCBreak(rc);
10884 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10885
10886 rc = VMXReadVmcs64(VMX_VMCS64_GUEST_PDPTE3_FULL, &u64Val);
10887 AssertRCBreak(rc);
10888 HMVMX_CHECK_BREAK(!(u64Val & X86_PDPE_PAE_MBZ_MASK), VMX_IGS_PAE_PDPTE_RESERVED);
10889 }
10890
10891 /* Shouldn't happen but distinguish it from AssertRCBreak() errors. */
10892 if (uError == VMX_IGS_ERROR)
10893 uError = VMX_IGS_REASON_NOT_FOUND;
10894 } while (0);
10895
10896 pVCpu->hm.s.u32HMError = uError;
10897 return uError;
10898
10899#undef HMVMX_ERROR_BREAK
10900#undef HMVMX_CHECK_BREAK
10901}
10902
10903/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10904/* -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- VM-exit handlers -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- */
10905/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-= */
10906
10907/** @name VM-exit handlers.
10908 * @{
10909 */
10910
10911/**
10912 * VM-exit handler for external interrupts (VMX_EXIT_EXT_INT).
10913 */
10914HMVMX_EXIT_DECL hmR0VmxExitExtInt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10915{
10916 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10917 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitExtInt);
10918 /* Windows hosts (32-bit and 64-bit) have DPC latency issues. See @bugref{6853}. */
10919 if (VMMR0ThreadCtxHookIsEnabled(pVCpu))
10920 return VINF_SUCCESS;
10921 return VINF_EM_RAW_INTERRUPT;
10922}
10923
10924
10925/**
10926 * VM-exit handler for exceptions or NMIs (VMX_EXIT_XCPT_OR_NMI).
10927 */
10928HMVMX_EXIT_DECL hmR0VmxExitXcptOrNmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
10929{
10930 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
10931 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitXcptNmi, y3);
10932
10933 int rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
10934 AssertRCReturn(rc, rc);
10935
10936 uint32_t uIntType = VMX_EXIT_INTERRUPTION_INFO_TYPE(pVmxTransient->uExitIntInfo);
10937 Assert( !(pVCpu->hm.s.vmx.u32ExitCtls & VMX_VMCS_CTRL_EXIT_ACK_EXT_INT)
10938 && uIntType != VMX_EXIT_INTERRUPTION_INFO_TYPE_EXT_INT);
10939 Assert(VMX_EXIT_INTERRUPTION_INFO_IS_VALID(pVmxTransient->uExitIntInfo));
10940
10941 if (uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_NMI)
10942 {
10943 /*
10944 * This cannot be a guest NMI as the only way for the guest to receive an NMI is if we
10945 * injected it ourselves and anything we inject is not going to cause a VM-exit directly
10946 * for the event being injected[1]. Go ahead and dispatch the NMI to the host[2].
10947 *
10948 * [1] -- See Intel spec. 27.2.3 "Information for VM Exits During Event Delivery".
10949 * [2] -- See Intel spec. 27.5.5 "Updating Non-Register State".
10950 */
10951 VMXDispatchHostNmi();
10952 STAM_REL_COUNTER_INC(&pVCpu->hm.s.StatExitHostNmiInGC);
10953 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10954 return VINF_SUCCESS;
10955 }
10956
10957 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
10958 VBOXSTRICTRC rcStrictRc1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
10959 if (RT_UNLIKELY(rcStrictRc1 == VINF_SUCCESS))
10960 { /* likely */ }
10961 else
10962 {
10963 if (rcStrictRc1 == VINF_HM_DOUBLE_FAULT)
10964 rcStrictRc1 = VINF_SUCCESS;
10965 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
10966 return rcStrictRc1;
10967 }
10968
10969 uint32_t uExitIntInfo = pVmxTransient->uExitIntInfo;
10970 uint32_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(uExitIntInfo);
10971 switch (uIntType)
10972 {
10973 case VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT: /* Privileged software exception. (#DB from ICEBP) */
10974 Assert(uVector == X86_XCPT_DB);
10975 RT_FALL_THRU();
10976 case VMX_EXIT_INTERRUPTION_INFO_TYPE_SW_XCPT: /* Software exception. (#BP or #OF) */
10977 Assert(uVector == X86_XCPT_BP || uVector == X86_XCPT_OF || uIntType == VMX_EXIT_INTERRUPTION_INFO_TYPE_PRIV_SW_XCPT);
10978 RT_FALL_THRU();
10979 case VMX_EXIT_INTERRUPTION_INFO_TYPE_HW_XCPT:
10980 {
10981 /*
10982 * If there's any exception caused as a result of event injection, the resulting
10983 * secondary/final execption will be pending, we shall continue guest execution
10984 * after injecting the event. The page-fault case is complicated and we manually
10985 * handle any currently pending event in hmR0VmxExitXcptPF.
10986 */
10987 if (!pVCpu->hm.s.Event.fPending)
10988 { /* likely */ }
10989 else if (uVector != X86_XCPT_PF)
10990 {
10991 rc = VINF_SUCCESS;
10992 break;
10993 }
10994
10995 switch (uVector)
10996 {
10997 case X86_XCPT_PF: rc = hmR0VmxExitXcptPF(pVCpu, pMixedCtx, pVmxTransient); break;
10998 case X86_XCPT_GP: rc = hmR0VmxExitXcptGP(pVCpu, pMixedCtx, pVmxTransient); break;
10999 case X86_XCPT_MF: rc = hmR0VmxExitXcptMF(pVCpu, pMixedCtx, pVmxTransient); break;
11000 case X86_XCPT_DB: rc = hmR0VmxExitXcptDB(pVCpu, pMixedCtx, pVmxTransient); break;
11001 case X86_XCPT_BP: rc = hmR0VmxExitXcptBP(pVCpu, pMixedCtx, pVmxTransient); break;
11002 case X86_XCPT_AC: rc = hmR0VmxExitXcptAC(pVCpu, pMixedCtx, pVmxTransient); break;
11003
11004 case X86_XCPT_NM: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNM);
11005 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11006 case X86_XCPT_XF: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXF);
11007 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11008 case X86_XCPT_DE: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDE);
11009 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11010 case X86_XCPT_UD: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestUD);
11011 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11012 case X86_XCPT_SS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestSS);
11013 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11014 case X86_XCPT_NP: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestNP);
11015 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11016 case X86_XCPT_TS: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestTS);
11017 rc = hmR0VmxExitXcptGeneric(pVCpu, pMixedCtx, pVmxTransient); break;
11018 default:
11019 {
11020 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestXcpUnk);
11021 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
11022 {
11023 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.vmx.pRealModeTSS);
11024 Assert(PDMVmmDevHeapIsEnabled(pVCpu->CTX_SUFF(pVM)));
11025 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
11026
11027 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
11028 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11029 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
11030 AssertRCReturn(rc, rc);
11031 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(uExitIntInfo),
11032 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode,
11033 0 /* GCPtrFaultAddress */);
11034 }
11035 else
11036 {
11037 AssertMsgFailed(("Unexpected VM-exit caused by exception %#x\n", uVector));
11038 pVCpu->hm.s.u32HMError = uVector;
11039 rc = VERR_VMX_UNEXPECTED_EXCEPTION;
11040 }
11041 break;
11042 }
11043 }
11044 break;
11045 }
11046
11047 default:
11048 {
11049 pVCpu->hm.s.u32HMError = uExitIntInfo;
11050 rc = VERR_VMX_UNEXPECTED_INTERRUPTION_EXIT_TYPE;
11051 AssertMsgFailed(("Unexpected interruption info %#x\n", VMX_EXIT_INTERRUPTION_INFO_TYPE(uExitIntInfo)));
11052 break;
11053 }
11054 }
11055 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitXcptNmi, y3);
11056 return rc;
11057}
11058
11059
11060/**
11061 * VM-exit handler for interrupt-window exiting (VMX_EXIT_INT_WINDOW).
11062 */
11063HMVMX_EXIT_NSRC_DECL hmR0VmxExitIntWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11064{
11065 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11066
11067 /* Indicate that we no longer need to VM-exit when the guest is ready to receive interrupts, it is now ready. */
11068 hmR0VmxClearIntWindowExitVmcs(pVCpu);
11069
11070 /* Deliver the pending interrupts via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11071 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIntWindow);
11072 return VINF_SUCCESS;
11073}
11074
11075
11076/**
11077 * VM-exit handler for NMI-window exiting (VMX_EXIT_NMI_WINDOW).
11078 */
11079HMVMX_EXIT_NSRC_DECL hmR0VmxExitNmiWindow(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11080{
11081 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11082 if (RT_UNLIKELY(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_NMI_WINDOW_EXIT)))
11083 {
11084 AssertMsgFailed(("Unexpected NMI-window exit.\n"));
11085 HMVMX_RETURN_UNEXPECTED_EXIT();
11086 }
11087
11088 Assert(!VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_BLOCK_NMIS));
11089
11090 /*
11091 * If block-by-STI is set when we get this VM-exit, it means the CPU doesn't block NMIs following STI.
11092 * It is therefore safe to unblock STI and deliver the NMI ourselves. See @bugref{7445}.
11093 */
11094 uint32_t fIntrState = 0;
11095 int rc = VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11096 AssertRCReturn(rc, rc);
11097
11098 bool const fBlockSti = RT_BOOL(fIntrState & VMX_VMCS_GUEST_INTERRUPTIBILITY_STATE_BLOCK_STI);
11099 if ( fBlockSti
11100 && VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS))
11101 {
11102 VMCPU_FF_CLEAR(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS);
11103 }
11104
11105 /* Indicate that we no longer need to VM-exit when the guest is ready to receive NMIs, it is now ready */
11106 hmR0VmxClearNmiWindowExitVmcs(pVCpu);
11107
11108 /* Deliver the pending NMI via hmR0VmxEvaluatePendingEvent() and resume guest execution. */
11109 return VINF_SUCCESS;
11110}
11111
11112
11113/**
11114 * VM-exit handler for WBINVD (VMX_EXIT_WBINVD). Conditional VM-exit.
11115 */
11116HMVMX_EXIT_NSRC_DECL hmR0VmxExitWbinvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11117{
11118 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11119 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11120}
11121
11122
11123/**
11124 * VM-exit handler for INVD (VMX_EXIT_INVD). Unconditional VM-exit.
11125 */
11126HMVMX_EXIT_NSRC_DECL hmR0VmxExitInvd(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11127{
11128 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11129 return hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11130}
11131
11132
11133/**
11134 * VM-exit handler for CPUID (VMX_EXIT_CPUID). Unconditional VM-exit.
11135 */
11136HMVMX_EXIT_DECL hmR0VmxExitCpuid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11137{
11138 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11139 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11140
11141 /*
11142 * Get the state we need and update the exit history entry.
11143 */
11144 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11145 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11146 | CPUMCTX_EXTRN_CS);
11147 AssertRCReturn(rc, rc);
11148
11149 VBOXSTRICTRC rcStrict;
11150 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
11151 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_CPUID),
11152 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
11153 if (!pExitRec)
11154 {
11155 /*
11156 * Regular CPUID instruction execution.
11157 */
11158 PVM pVM = pVCpu->CTX_SUFF(pVM);
11159 rcStrict = EMInterpretCpuId(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11160 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11161 {
11162 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11163 Assert(pVmxTransient->cbInstr == 2);
11164 }
11165 else
11166 {
11167 AssertMsgFailed(("hmR0VmxExitCpuid: EMInterpretCpuId failed with %Rrc\n", rc));
11168 rcStrict = VERR_EM_INTERPRETER;
11169 }
11170 }
11171 else
11172 {
11173 /*
11174 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
11175 */
11176 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
11177 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11178 AssertRCReturn(rc2, rc2);
11179
11180 Log4(("CpuIdExit/%u: %04x:%08RX64: %#x/%#x -> EMHistoryExec\n",
11181 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, pVCpu->cpum.GstCtx.eax, pVCpu->cpum.GstCtx.ecx));
11182
11183 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
11184 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
11185
11186 Log4(("CpuIdExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
11187 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
11188 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
11189 }
11190 return VBOXSTRICTRC_TODO(rcStrict);
11191}
11192
11193
11194/**
11195 * VM-exit handler for GETSEC (VMX_EXIT_GETSEC). Unconditional VM-exit.
11196 */
11197HMVMX_EXIT_DECL hmR0VmxExitGetsec(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11198{
11199 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11200 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4);
11201 AssertRCReturn(rc, rc);
11202
11203 if (pMixedCtx->cr4 & X86_CR4_SMXE)
11204 return VINF_EM_RAW_EMULATE_INSTR;
11205
11206 AssertMsgFailed(("hmR0VmxExitGetsec: unexpected VM-exit when CR4.SMXE is 0.\n"));
11207 HMVMX_RETURN_UNEXPECTED_EXIT();
11208}
11209
11210
11211/**
11212 * VM-exit handler for RDTSC (VMX_EXIT_RDTSC). Conditional VM-exit.
11213 */
11214HMVMX_EXIT_DECL hmR0VmxExitRdtsc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11215{
11216 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11217 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11218 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11219 AssertRCReturn(rc, rc);
11220
11221 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtsc(pVCpu, pVmxTransient->cbInstr);
11222 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11223 {
11224 /* If we get a spurious VM-exit when offsetting is enabled,
11225 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11226 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11227 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11228 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11229 | HM_CHANGED_GUEST_RFLAGS);
11230 }
11231 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11232 {
11233 rcStrict = VINF_SUCCESS;
11234 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11235 }
11236 return rcStrict;
11237}
11238
11239
11240/**
11241 * VM-exit handler for RDTSCP (VMX_EXIT_RDTSCP). Conditional VM-exit.
11242 */
11243HMVMX_EXIT_DECL hmR0VmxExitRdtscp(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11244{
11245 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11246 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11247 | CPUMCTX_EXTRN_TSC_AUX);
11248 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11249 AssertRCReturn(rc, rc);
11250
11251 VBOXSTRICTRC rcStrict = IEMExecDecodedRdtscp(pVCpu, pVmxTransient->cbInstr);
11252 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11253 {
11254 /* If we get a spurious VM-exit when offsetting is enabled,
11255 we must reset offsetting on VM-reentry. See @bugref{6634}. */
11256 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TSC_OFFSETTING)
11257 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11258 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
11259 | HM_CHANGED_GUEST_RFLAGS);
11260 }
11261 else if (rcStrict == VINF_IEM_RAISED_XCPT)
11262 {
11263 rcStrict = VINF_SUCCESS;
11264 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_XCPT_RAISED_MASK);
11265 }
11266 return rcStrict;
11267}
11268
11269
11270/**
11271 * VM-exit handler for RDPMC (VMX_EXIT_RDPMC). Conditional VM-exit.
11272 */
11273HMVMX_EXIT_DECL hmR0VmxExitRdpmc(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11274{
11275 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11276 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR4
11277 | CPUMCTX_EXTRN_CR0
11278 | CPUMCTX_EXTRN_RFLAGS
11279 | CPUMCTX_EXTRN_SS);
11280 AssertRCReturn(rc, rc);
11281
11282 PVM pVM = pVCpu->CTX_SUFF(pVM);
11283 rc = EMInterpretRdpmc(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11284 if (RT_LIKELY(rc == VINF_SUCCESS))
11285 {
11286 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11287 Assert(pVmxTransient->cbInstr == 2);
11288 }
11289 else
11290 {
11291 AssertMsgFailed(("hmR0VmxExitRdpmc: EMInterpretRdpmc failed with %Rrc\n", rc));
11292 rc = VERR_EM_INTERPRETER;
11293 }
11294 return rc;
11295}
11296
11297
11298/**
11299 * VM-exit handler for VMCALL (VMX_EXIT_VMCALL). Unconditional VM-exit.
11300 */
11301HMVMX_EXIT_DECL hmR0VmxExitVmcall(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11302{
11303 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11304
11305 VBOXSTRICTRC rcStrict = VERR_VMX_IPE_3;
11306 if (EMAreHypercallInstructionsEnabled(pVCpu))
11307 {
11308 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_RIP
11309 | CPUMCTX_EXTRN_RFLAGS
11310 | CPUMCTX_EXTRN_CR0
11311 | CPUMCTX_EXTRN_SS
11312 | CPUMCTX_EXTRN_CS
11313 | CPUMCTX_EXTRN_EFER);
11314 AssertRCReturn(rc, rc);
11315
11316 /* Perform the hypercall. */
11317 rcStrict = GIMHypercall(pVCpu, pMixedCtx);
11318 if (rcStrict == VINF_SUCCESS)
11319 {
11320 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11321 AssertRCReturn(rc, rc);
11322 }
11323 else
11324 Assert( rcStrict == VINF_GIM_R3_HYPERCALL
11325 || rcStrict == VINF_GIM_HYPERCALL_CONTINUING
11326 || RT_FAILURE(rcStrict));
11327
11328 /* If the hypercall changes anything other than guest's general-purpose registers,
11329 we would need to reload the guest changed bits here before VM-entry. */
11330 }
11331 else
11332 Log4Func(("Hypercalls not enabled\n"));
11333
11334 /* If hypercalls are disabled or the hypercall failed for some reason, raise #UD and continue. */
11335 if (RT_FAILURE(rcStrict))
11336 {
11337 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11338 rcStrict = VINF_SUCCESS;
11339 }
11340
11341 return rcStrict;
11342}
11343
11344
11345/**
11346 * VM-exit handler for INVLPG (VMX_EXIT_INVLPG). Conditional VM-exit.
11347 */
11348HMVMX_EXIT_DECL hmR0VmxExitInvlpg(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11349{
11350 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11351 PVM pVM = pVCpu->CTX_SUFF(pVM);
11352 Assert(!pVM->hm.s.fNestedPaging || pVCpu->hm.s.fUsingDebugLoop);
11353
11354 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11355 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK);
11356 AssertRCReturn(rc, rc);
11357
11358 VBOXSTRICTRC rcStrict = EMInterpretInvlpg(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx), pVmxTransient->uExitQualification);
11359 if (RT_LIKELY(rcStrict == VINF_SUCCESS))
11360 rcStrict = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11361 else
11362 AssertMsg(rcStrict == VERR_EM_INTERPRETER, ("hmR0VmxExitInvlpg: EMInterpretInvlpg %#RX64 failed with %Rrc\n",
11363 pVmxTransient->uExitQualification, VBOXSTRICTRC_VAL(rcStrict)));
11364 return rcStrict;
11365}
11366
11367
11368/**
11369 * VM-exit handler for MONITOR (VMX_EXIT_MONITOR). Conditional VM-exit.
11370 */
11371HMVMX_EXIT_DECL hmR0VmxExitMonitor(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11372{
11373 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11374 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11375 | CPUMCTX_EXTRN_RFLAGS
11376 | CPUMCTX_EXTRN_SS);
11377 AssertRCReturn(rc, rc);
11378
11379 PVM pVM = pVCpu->CTX_SUFF(pVM);
11380 rc = EMInterpretMonitor(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11381 if (RT_LIKELY(rc == VINF_SUCCESS))
11382 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11383 else
11384 {
11385 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMonitor: EMInterpretMonitor failed with %Rrc\n", rc));
11386 rc = VERR_EM_INTERPRETER;
11387 }
11388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMonitor);
11389 return rc;
11390}
11391
11392
11393/**
11394 * VM-exit handler for MWAIT (VMX_EXIT_MWAIT). Conditional VM-exit.
11395 */
11396HMVMX_EXIT_DECL hmR0VmxExitMwait(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11397{
11398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11399 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11400 | CPUMCTX_EXTRN_RFLAGS
11401 | CPUMCTX_EXTRN_SS);
11402 AssertRCReturn(rc, rc);
11403
11404 PVM pVM = pVCpu->CTX_SUFF(pVM);
11405 VBOXSTRICTRC rc2 = EMInterpretMWait(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11406 rc = VBOXSTRICTRC_VAL(rc2);
11407 if (RT_LIKELY( rc == VINF_SUCCESS
11408 || rc == VINF_EM_HALT))
11409 {
11410 int rc3 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11411 AssertRCReturn(rc3, rc3);
11412
11413 if ( rc == VINF_EM_HALT
11414 && EMMonitorWaitShouldContinue(pVCpu, pMixedCtx))
11415 rc = VINF_SUCCESS;
11416 }
11417 else
11418 {
11419 AssertMsg(rc == VERR_EM_INTERPRETER, ("hmR0VmxExitMwait: EMInterpretMWait failed with %Rrc\n", rc));
11420 rc = VERR_EM_INTERPRETER;
11421 }
11422 AssertMsg(rc == VINF_SUCCESS || rc == VINF_EM_HALT || rc == VERR_EM_INTERPRETER,
11423 ("hmR0VmxExitMwait: failed, invalid error code %Rrc\n", rc));
11424 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMwait);
11425 return rc;
11426}
11427
11428
11429/**
11430 * VM-exit handler for RSM (VMX_EXIT_RSM). Unconditional VM-exit.
11431 */
11432HMVMX_EXIT_NSRC_DECL hmR0VmxExitRsm(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11433{
11434 /*
11435 * Execution of RSM outside of SMM mode causes #UD regardless of VMX root or VMX non-root
11436 * mode. In theory, we should never get this VM-exit. This can happen only if dual-monitor
11437 * treatment of SMI and VMX is enabled, which can (only?) be done by executing VMCALL in
11438 * VMX root operation. If we get here, something funny is going on.
11439 *
11440 * See Intel spec. 33.15.5 "Enabling the Dual-Monitor Treatment".
11441 */
11442 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11443 AssertMsgFailed(("Unexpected RSM VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11444 HMVMX_RETURN_UNEXPECTED_EXIT();
11445}
11446
11447
11448/**
11449 * VM-exit handler for SMI (VMX_EXIT_SMI). Unconditional VM-exit.
11450 */
11451HMVMX_EXIT_NSRC_DECL hmR0VmxExitSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11452{
11453 /*
11454 * This can only happen if we support dual-monitor treatment of SMI, which can be activated
11455 * by executing VMCALL in VMX root operation. Only an STM (SMM transfer monitor) would get
11456 * this VM-exit when we (the executive monitor) execute a VMCALL in VMX root mode or receive
11457 * an SMI. If we get here, something funny is going on.
11458 *
11459 * See Intel spec. 33.15.6 "Activating the Dual-Monitor Treatment"
11460 * See Intel spec. 25.3 "Other Causes of VM-Exits"
11461 */
11462 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11463 AssertMsgFailed(("Unexpected SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11464 HMVMX_RETURN_UNEXPECTED_EXIT();
11465}
11466
11467
11468/**
11469 * VM-exit handler for IO SMI (VMX_EXIT_IO_SMI). Unconditional VM-exit.
11470 */
11471HMVMX_EXIT_NSRC_DECL hmR0VmxExitIoSmi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11472{
11473 /* Same treatment as VMX_EXIT_SMI. See comment in hmR0VmxExitSmi(). */
11474 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11475 AssertMsgFailed(("Unexpected IO SMI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11476 HMVMX_RETURN_UNEXPECTED_EXIT();
11477}
11478
11479
11480/**
11481 * VM-exit handler for SIPI (VMX_EXIT_SIPI). Conditional VM-exit.
11482 */
11483HMVMX_EXIT_NSRC_DECL hmR0VmxExitSipi(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11484{
11485 /*
11486 * SIPI exits can only occur in VMX non-root operation when the "wait-for-SIPI" guest activity state is used.
11487 * We don't make use of it as our guests don't have direct access to the host LAPIC.
11488 * See Intel spec. 25.3 "Other Causes of VM-exits".
11489 */
11490 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11491 AssertMsgFailed(("Unexpected SIPI VM-exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11492 HMVMX_RETURN_UNEXPECTED_EXIT();
11493}
11494
11495
11496/**
11497 * VM-exit handler for INIT signal (VMX_EXIT_INIT_SIGNAL). Unconditional
11498 * VM-exit.
11499 */
11500HMVMX_EXIT_NSRC_DECL hmR0VmxExitInitSignal(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11501{
11502 /*
11503 * INIT signals are blocked in VMX root operation by VMXON and by SMI in SMM.
11504 * See Intel spec. 33.14.1 Default Treatment of SMI Delivery" and Intel spec. 29.3 "VMX Instructions" for "VMXON".
11505 *
11506 * It is -NOT- blocked in VMX non-root operation so we can, in theory, still get these VM-exits.
11507 * See Intel spec. "23.8 Restrictions on VMX operation".
11508 */
11509 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11510 return VINF_SUCCESS;
11511}
11512
11513
11514/**
11515 * VM-exit handler for triple faults (VMX_EXIT_TRIPLE_FAULT). Unconditional
11516 * VM-exit.
11517 */
11518HMVMX_EXIT_DECL hmR0VmxExitTripleFault(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11519{
11520 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11521 return VINF_EM_RESET;
11522}
11523
11524
11525/**
11526 * VM-exit handler for HLT (VMX_EXIT_HLT). Conditional VM-exit.
11527 */
11528HMVMX_EXIT_DECL hmR0VmxExitHlt(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11529{
11530 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11531 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_HLT_EXIT);
11532
11533 int rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11534 AssertRCReturn(rc, rc);
11535
11536 if (EMShouldContinueAfterHalt(pVCpu, pMixedCtx)) /* Requires eflags. */
11537 rc = VINF_SUCCESS;
11538 else
11539 rc = VINF_EM_HALT;
11540
11541 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
11542 if (rc != VINF_SUCCESS)
11543 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchHltToR3);
11544 return rc;
11545}
11546
11547
11548/**
11549 * VM-exit handler for instructions that result in a \#UD exception delivered to
11550 * the guest.
11551 */
11552HMVMX_EXIT_NSRC_DECL hmR0VmxExitSetPendingXcptUD(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11553{
11554 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11555 hmR0VmxSetPendingXcptUD(pVCpu, pMixedCtx);
11556 return VINF_SUCCESS;
11557}
11558
11559
11560/**
11561 * VM-exit handler for expiry of the VMX preemption timer.
11562 */
11563HMVMX_EXIT_DECL hmR0VmxExitPreemptTimer(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11564{
11565 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11566
11567 /* If the preemption-timer has expired, reinitialize the preemption timer on next VM-entry. */
11568 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11569
11570 /* If there are any timer events pending, fall back to ring-3, otherwise resume guest execution. */
11571 PVM pVM = pVCpu->CTX_SUFF(pVM);
11572 bool fTimersPending = TMTimerPollBool(pVM, pVCpu);
11573 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPreemptTimer);
11574 return fTimersPending ? VINF_EM_RAW_TIMER_PENDING : VINF_SUCCESS;
11575}
11576
11577
11578/**
11579 * VM-exit handler for XSETBV (VMX_EXIT_XSETBV). Unconditional VM-exit.
11580 */
11581HMVMX_EXIT_DECL hmR0VmxExitXsetbv(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11582{
11583 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11584
11585 int rc = hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11586 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
11587 | CPUMCTX_EXTRN_CR4);
11588 AssertRCReturn(rc, rc);
11589
11590 VBOXSTRICTRC rcStrict = IEMExecDecodedXsetbv(pVCpu, pVmxTransient->cbInstr);
11591 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
11592 : HM_CHANGED_XCPT_RAISED_MASK);
11593
11594 pVCpu->hm.s.fLoadSaveGuestXcr0 = (pMixedCtx->cr4 & X86_CR4_OSXSAVE) && pMixedCtx->aXcr[0] != ASMGetXcr0();
11595
11596 return rcStrict;
11597}
11598
11599
11600/**
11601 * VM-exit handler for INVPCID (VMX_EXIT_INVPCID). Conditional VM-exit.
11602 */
11603HMVMX_EXIT_DECL hmR0VmxExitInvpcid(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11604{
11605 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11606 /** @todo Use VM-exit instruction information. */
11607 return VERR_EM_INTERPRETER;
11608}
11609
11610
11611/**
11612 * VM-exit handler for invalid-guest-state (VMX_EXIT_ERR_INVALID_GUEST_STATE).
11613 * Error VM-exit.
11614 */
11615HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrInvalidGuestState(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11616{
11617 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
11618 rc |= hmR0VmxCheckVmcsCtls(pVCpu);
11619 AssertRCReturn(rc, rc);
11620
11621 uint32_t uInvalidReason = hmR0VmxCheckGuestState(pVCpu, pMixedCtx);
11622 NOREF(uInvalidReason);
11623
11624#ifdef VBOX_STRICT
11625 uint32_t fIntrState;
11626 RTHCUINTREG uHCReg;
11627 uint64_t u64Val;
11628 uint32_t u32Val;
11629
11630 rc = hmR0VmxReadEntryIntInfoVmcs(pVmxTransient);
11631 rc |= hmR0VmxReadEntryXcptErrorCodeVmcs(pVmxTransient);
11632 rc |= hmR0VmxReadEntryInstrLenVmcs(pVmxTransient);
11633 rc |= VMXReadVmcs32(VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE, &fIntrState);
11634 AssertRCReturn(rc, rc);
11635
11636 Log4(("uInvalidReason %u\n", uInvalidReason));
11637 Log4(("VMX_VMCS32_CTRL_ENTRY_INTERRUPTION_INFO %#RX32\n", pVmxTransient->uEntryIntInfo));
11638 Log4(("VMX_VMCS32_CTRL_ENTRY_EXCEPTION_ERRCODE %#RX32\n", pVmxTransient->uEntryXcptErrorCode));
11639 Log4(("VMX_VMCS32_CTRL_ENTRY_INSTR_LENGTH %#RX32\n", pVmxTransient->cbEntryInstr));
11640 Log4(("VMX_VMCS32_GUEST_INTERRUPTIBILITY_STATE %#RX32\n", fIntrState));
11641
11642 rc = VMXReadVmcs32(VMX_VMCS_GUEST_CR0, &u32Val); AssertRC(rc);
11643 Log4(("VMX_VMCS_GUEST_CR0 %#RX32\n", u32Val));
11644 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_MASK, &uHCReg); AssertRC(rc);
11645 Log4(("VMX_VMCS_CTRL_CR0_MASK %#RHr\n", uHCReg));
11646 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR0_READ_SHADOW, &uHCReg); AssertRC(rc);
11647 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11648 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_MASK, &uHCReg); AssertRC(rc);
11649 Log4(("VMX_VMCS_CTRL_CR4_MASK %#RHr\n", uHCReg));
11650 rc = VMXReadVmcsHstN(VMX_VMCS_CTRL_CR4_READ_SHADOW, &uHCReg); AssertRC(rc);
11651 Log4(("VMX_VMCS_CTRL_CR4_READ_SHADOW %#RHr\n", uHCReg));
11652 rc = VMXReadVmcs64(VMX_VMCS64_CTRL_EPTP_FULL, &u64Val); AssertRC(rc);
11653 Log4(("VMX_VMCS64_CTRL_EPTP_FULL %#RX64\n", u64Val));
11654
11655 hmR0DumpRegs(pVCpu, pMixedCtx);
11656#else
11657 NOREF(pVmxTransient);
11658#endif
11659
11660 return VERR_VMX_INVALID_GUEST_STATE;
11661}
11662
11663
11664/**
11665 * VM-exit handler for VM-entry failure due to an MSR-load
11666 * (VMX_EXIT_ERR_MSR_LOAD). Error VM-exit.
11667 */
11668HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMsrLoad(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11669{
11670 NOREF(pVmxTransient);
11671 AssertMsgFailed(("Unexpected MSR-load exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11672 HMVMX_RETURN_UNEXPECTED_EXIT();
11673}
11674
11675
11676/**
11677 * VM-exit handler for VM-entry failure due to a machine-check event
11678 * (VMX_EXIT_ERR_MACHINE_CHECK). Error VM-exit.
11679 */
11680HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrMachineCheck(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11681{
11682 NOREF(pVmxTransient);
11683 AssertMsgFailed(("Unexpected machine-check event exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx)); NOREF(pMixedCtx);
11684 HMVMX_RETURN_UNEXPECTED_EXIT();
11685}
11686
11687
11688/**
11689 * VM-exit handler for all undefined reasons. Should never ever happen.. in
11690 * theory.
11691 */
11692HMVMX_EXIT_NSRC_DECL hmR0VmxExitErrUndefined(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11693{
11694 AssertMsgFailed(("Huh!? Undefined VM-exit reason %d. pVCpu=%p pMixedCtx=%p\n", pVmxTransient->uExitReason, pVCpu, pMixedCtx));
11695 NOREF(pVCpu); NOREF(pMixedCtx); NOREF(pVmxTransient);
11696 return VERR_VMX_UNDEFINED_EXIT_CODE;
11697}
11698
11699
11700/**
11701 * VM-exit handler for XDTR (LGDT, SGDT, LIDT, SIDT) accesses
11702 * (VMX_EXIT_XDTR_ACCESS) and LDT and TR access (LLDT, LTR, SLDT, STR).
11703 * Conditional VM-exit.
11704 */
11705HMVMX_EXIT_DECL hmR0VmxExitXdtrAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11706{
11707 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11708
11709 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT. */
11710 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitXdtrAccess);
11711 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_DESCRIPTOR_TABLE_EXIT)
11712 return VERR_EM_INTERPRETER;
11713 AssertMsgFailed(("Unexpected XDTR access. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11714 HMVMX_RETURN_UNEXPECTED_EXIT();
11715}
11716
11717
11718/**
11719 * VM-exit handler for RDRAND (VMX_EXIT_RDRAND). Conditional VM-exit.
11720 */
11721HMVMX_EXIT_DECL hmR0VmxExitRdrand(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11722{
11723 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11724
11725 /* By default, we don't enable VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT. */
11726 if (pVCpu->hm.s.vmx.u32ProcCtls2 & VMX_VMCS_CTRL_PROC_EXEC2_RDRAND_EXIT)
11727 return VERR_EM_INTERPRETER;
11728 AssertMsgFailed(("Unexpected RDRAND exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
11729 HMVMX_RETURN_UNEXPECTED_EXIT();
11730}
11731
11732
11733/**
11734 * VM-exit handler for RDMSR (VMX_EXIT_RDMSR).
11735 */
11736HMVMX_EXIT_DECL hmR0VmxExitRdmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11737{
11738 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11739
11740 /* EMInterpretRdmsr() requires CR0, Eflags and SS segment register. */
11741 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11742 | CPUMCTX_EXTRN_RFLAGS
11743 | CPUMCTX_EXTRN_SS);
11744 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11745 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11746 AssertRCReturn(rc, rc);
11747 Log4Func(("ecx=%#RX32\n", pMixedCtx->ecx));
11748
11749#ifdef VBOX_STRICT
11750 if (pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS)
11751 {
11752 if ( hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx)
11753 && pMixedCtx->ecx != MSR_K6_EFER)
11754 {
11755 AssertMsgFailed(("Unexpected RDMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11756 pMixedCtx->ecx));
11757 HMVMX_RETURN_UNEXPECTED_EXIT();
11758 }
11759 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11760 {
11761 VMXMSREXITREAD enmRead;
11762 VMXMSREXITWRITE enmWrite;
11763 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11764 AssertRCReturn(rc2, rc2);
11765 if (enmRead == VMXMSREXIT_PASSTHRU_READ)
11766 {
11767 AssertMsgFailed(("Unexpected RDMSR for a passthru lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11768 HMVMX_RETURN_UNEXPECTED_EXIT();
11769 }
11770 }
11771 }
11772#endif
11773
11774 PVM pVM = pVCpu->CTX_SUFF(pVM);
11775 rc = EMInterpretRdmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11776 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER,
11777 ("hmR0VmxExitRdmsr: failed, invalid error code %Rrc\n", rc));
11778 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitRdmsr);
11779 if (RT_SUCCESS(rc))
11780 {
11781 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11782 Assert(pVmxTransient->cbInstr == 2);
11783 }
11784 return rc;
11785}
11786
11787
11788/**
11789 * VM-exit handler for WRMSR (VMX_EXIT_WRMSR).
11790 */
11791HMVMX_EXIT_DECL hmR0VmxExitWrmsr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11792{
11793 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11794 PVM pVM = pVCpu->CTX_SUFF(pVM);
11795 int rc = VINF_SUCCESS;
11796
11797 /* EMInterpretWrmsr() requires CR0, EFLAGS and SS segment register. */
11798 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0
11799 | CPUMCTX_EXTRN_RFLAGS
11800 | CPUMCTX_EXTRN_SS);
11801 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11802 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_ALL_MSRS);
11803 AssertRCReturn(rc, rc);
11804 Log4Func(("ecx=%#RX32 edx:eax=%#RX32:%#RX32\n", pMixedCtx->ecx, pMixedCtx->edx, pMixedCtx->eax));
11805
11806 rc = EMInterpretWrmsr(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx));
11807 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER, ("hmR0VmxExitWrmsr: failed, invalid error code %Rrc\n", rc));
11808 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitWrmsr);
11809
11810 if (RT_SUCCESS(rc))
11811 {
11812 rc = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
11813
11814 /* If this is an X2APIC WRMSR access, update the APIC state as well. */
11815 if ( pMixedCtx->ecx == MSR_IA32_APICBASE
11816 || ( pMixedCtx->ecx >= MSR_IA32_X2APIC_START
11817 && pMixedCtx->ecx <= MSR_IA32_X2APIC_END))
11818 {
11819 /*
11820 * We've already saved the APIC related guest-state (TPR) in hmR0VmxPostRunGuest(). When full APIC register
11821 * virtualization is implemented we'll have to make sure APIC state is saved from the VMCS before
11822 * EMInterpretWrmsr() changes it.
11823 */
11824 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
11825 }
11826 else if (pMixedCtx->ecx == MSR_IA32_TSC) /* Windows 7 does this during bootup. See @bugref{6398}. */
11827 pVmxTransient->fUpdateTscOffsettingAndPreemptTimer = true;
11828 else if (pMixedCtx->ecx == MSR_K6_EFER)
11829 {
11830 /*
11831 * If the guest touches EFER we need to update the VM-Entry and VM-Exit controls as well,
11832 * even if it is -not- touching bits that cause paging mode changes (LMA/LME). We care about
11833 * the other bits as well, SCE and NXE. See @bugref{7368}.
11834 */
11835 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_EFER_MSR
11836 | HM_CHANGED_VMX_ENTRY_CTLS
11837 | HM_CHANGED_VMX_EXIT_CTLS);
11838 }
11839
11840 /* Update MSRs that are part of the VMCS and auto-load/store area when MSR-bitmaps are not supported. */
11841 if (!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_MSR_BITMAPS))
11842 {
11843 switch (pMixedCtx->ecx)
11844 {
11845 case MSR_IA32_SYSENTER_CS: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_CS_MSR); break;
11846 case MSR_IA32_SYSENTER_EIP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_EIP_MSR); break;
11847 case MSR_IA32_SYSENTER_ESP: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_SYSENTER_ESP_MSR); break;
11848 case MSR_K8_FS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_FS); break;
11849 case MSR_K8_GS_BASE: ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_GS); break;
11850 case MSR_K6_EFER: /* Nothing to do, already handled above. */ break;
11851 default:
11852 {
11853 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11854 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_AUTO_MSRS);
11855 else if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11856 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_VMX_GUEST_LAZY_MSRS);
11857 break;
11858 }
11859 }
11860 }
11861#ifdef VBOX_STRICT
11862 else
11863 {
11864 /* Paranoia. Validate that MSRs in the MSR-bitmaps with write-passthru are not intercepted. */
11865 switch (pMixedCtx->ecx)
11866 {
11867 case MSR_IA32_SYSENTER_CS:
11868 case MSR_IA32_SYSENTER_EIP:
11869 case MSR_IA32_SYSENTER_ESP:
11870 case MSR_K8_FS_BASE:
11871 case MSR_K8_GS_BASE:
11872 {
11873 AssertMsgFailed(("Unexpected WRMSR for an MSR in the VMCS. ecx=%#RX32\n", pMixedCtx->ecx));
11874 HMVMX_RETURN_UNEXPECTED_EXIT();
11875 }
11876
11877 /* Writes to MSRs in auto-load/store area/swapped MSRs, shouldn't cause VM-exits with MSR-bitmaps. */
11878 default:
11879 {
11880 if (hmR0VmxIsAutoLoadStoreGuestMsr(pVCpu, pMixedCtx->ecx))
11881 {
11882 /* EFER writes are always intercepted, see hmR0VmxExportGuestMsrs(). */
11883 if (pMixedCtx->ecx != MSR_K6_EFER)
11884 {
11885 AssertMsgFailed(("Unexpected WRMSR for an MSR in the auto-load/store area in the VMCS. ecx=%#RX32\n",
11886 pMixedCtx->ecx));
11887 HMVMX_RETURN_UNEXPECTED_EXIT();
11888 }
11889 }
11890
11891 if (hmR0VmxIsLazyGuestMsr(pVCpu, pMixedCtx->ecx))
11892 {
11893 VMXMSREXITREAD enmRead;
11894 VMXMSREXITWRITE enmWrite;
11895 int rc2 = hmR0VmxGetMsrPermission(pVCpu, pMixedCtx->ecx, &enmRead, &enmWrite);
11896 AssertRCReturn(rc2, rc2);
11897 if (enmWrite == VMXMSREXIT_PASSTHRU_WRITE)
11898 {
11899 AssertMsgFailed(("Unexpected WRMSR for passthru, lazy-restore MSR. ecx=%#RX32\n", pMixedCtx->ecx));
11900 HMVMX_RETURN_UNEXPECTED_EXIT();
11901 }
11902 }
11903 break;
11904 }
11905 }
11906 }
11907#endif /* VBOX_STRICT */
11908 }
11909 return rc;
11910}
11911
11912
11913/**
11914 * VM-exit handler for PAUSE (VMX_EXIT_PAUSE). Conditional VM-exit.
11915 */
11916HMVMX_EXIT_DECL hmR0VmxExitPause(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11917{
11918 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11919 /** @todo The guest has likely hit a contended spinlock. We might want to
11920 * poke a schedule different guest VCPU. */
11921 return VINF_EM_RAW_INTERRUPT;
11922}
11923
11924
11925/**
11926 * VM-exit handler for when the TPR value is lowered below the specified
11927 * threshold (VMX_EXIT_TPR_BELOW_THRESHOLD). Conditional VM-exit.
11928 */
11929HMVMX_EXIT_NSRC_DECL hmR0VmxExitTprBelowThreshold(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11930{
11931 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11932 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW);
11933
11934 /*
11935 * The TPR shadow would've been synced with the APIC TPR in hmR0VmxPostRunGuest(). We'll re-evaluate
11936 * pending interrupts and inject them before the next VM-entry so we can just continue execution here.
11937 */
11938 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTprBelowThreshold);
11939 return VINF_SUCCESS;
11940}
11941
11942
11943/**
11944 * VM-exit handler for control-register accesses (VMX_EXIT_MOV_CRX). Conditional
11945 * VM-exit.
11946 *
11947 * @retval VINF_SUCCESS when guest execution can continue.
11948 * @retval VINF_PGM_CHANGE_MODE when shadow paging mode changed, back to ring-3.
11949 * @retval VINF_PGM_SYNC_CR3 CR3 sync is required, back to ring-3.
11950 * @retval VERR_EM_INTERPRETER when something unexpected happened, fallback to
11951 * interpreter.
11952 */
11953HMVMX_EXIT_DECL hmR0VmxExitMovCRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
11954{
11955 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
11956 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitMovCRx, y2);
11957
11958 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
11959 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
11960 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
11961 AssertRCReturn(rc, rc);
11962
11963 VBOXSTRICTRC rcStrict;
11964 PVM pVM = pVCpu->CTX_SUFF(pVM);
11965 RTGCUINTPTR const uExitQualification = pVmxTransient->uExitQualification;
11966 uint32_t const uAccessType = VMX_EXIT_QUAL_CRX_ACCESS(uExitQualification);
11967 switch (uAccessType)
11968 {
11969 case VMX_EXIT_QUAL_CRX_ACCESS_WRITE: /* MOV to CRx */
11970 {
11971 rcStrict = IEMExecDecodedMovCRxWrite(pVCpu, pVmxTransient->cbInstr,
11972 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
11973 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification));
11974 AssertMsg( rcStrict == VINF_SUCCESS
11975 || rcStrict == VINF_IEM_RAISED_XCPT
11976 || rcStrict == VINF_PGM_CHANGE_MODE
11977 || rcStrict == VINF_PGM_SYNC_CR3, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
11978
11979 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
11980 {
11981 case 0:
11982 {
11983 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
11984 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Write);
11985 Log4(("CRX CR0 write rcStrict=%Rrc CR0=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr0));
11986 break;
11987 }
11988
11989 case 2:
11990 {
11991 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Write);
11992 /* Nothing to do here, CR2 it's not part of the VMCS. */
11993 break;
11994 }
11995
11996 case 3:
11997 {
11998 Assert(!pVM->hm.s.fNestedPaging || !CPUMIsGuestPagingEnabledEx(pMixedCtx) || pVCpu->hm.s.fUsingDebugLoop);
11999 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Write);
12000 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR3);
12001 Log4(("CRX CR3 write rcStrict=%Rrc CR3=%#RX64\n", VBOXSTRICTRC_VAL(rcStrict), pMixedCtx->cr3));
12002 break;
12003 }
12004
12005 case 4:
12006 {
12007 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Write);
12008 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR4);
12009 Log4(("CRX CR4 write rc=%Rrc CR4=%#RX64 fLoadSaveGuestXcr0=%u\n", VBOXSTRICTRC_VAL(rcStrict),
12010 pMixedCtx->cr4, pVCpu->hm.s.fLoadSaveGuestXcr0));
12011 break;
12012 }
12013
12014 case 8:
12015 {
12016 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Write);
12017 Assert(!(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12018 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_APIC_TPR);
12019 break;
12020 }
12021 default:
12022 AssertMsgFailed(("Invalid CRx register %#x\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification)));
12023 break;
12024 }
12025 break;
12026 }
12027
12028 case VMX_EXIT_QUAL_CRX_ACCESS_READ: /* MOV from CRx */
12029 {
12030 Assert( !pVM->hm.s.fNestedPaging
12031 || !CPUMIsGuestPagingEnabledEx(pMixedCtx)
12032 || pVCpu->hm.s.fUsingDebugLoop
12033 || VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 3);
12034 /* CR8 reads only cause a VM-exit when the TPR shadow feature isn't enabled. */
12035 Assert( VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification) != 8
12036 || !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW));
12037
12038 rcStrict = IEMExecDecodedMovCRxRead(pVCpu, pVmxTransient->cbInstr,
12039 VMX_EXIT_QUAL_CRX_GENREG(uExitQualification),
12040 VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification));
12041 AssertMsg( rcStrict == VINF_SUCCESS
12042 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12043#ifdef VBOX_WITH_STATISTICS
12044 switch (VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification))
12045 {
12046 case 0: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR0Read); break;
12047 case 2: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR2Read); break;
12048 case 3: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR3Read); break;
12049 case 4: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR4Read); break;
12050 case 8: STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCR8Read); break;
12051 }
12052#endif
12053 Log4(("CRX CR%d Read access rcStrict=%Rrc\n", VMX_EXIT_QUAL_CRX_REGISTER(uExitQualification),
12054 VBOXSTRICTRC_VAL(rcStrict)));
12055 if (VMX_EXIT_QUAL_CRX_GENREG(uExitQualification) == X86_GREG_xSP)
12056 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RSP);
12057 break;
12058 }
12059
12060 case VMX_EXIT_QUAL_CRX_ACCESS_CLTS: /* CLTS (Clear Task-Switch Flag in CR0) */
12061 {
12062 rcStrict = IEMExecDecodedClts(pVCpu, pVmxTransient->cbInstr);
12063 AssertMsg( rcStrict == VINF_SUCCESS
12064 || rcStrict == VINF_IEM_RAISED_XCPT, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12065
12066 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12067 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitClts);
12068 Log4(("CRX CLTS rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12069 break;
12070 }
12071
12072 case VMX_EXIT_QUAL_CRX_ACCESS_LMSW: /* LMSW (Load Machine-Status Word into CR0) */
12073 {
12074 rcStrict = IEMExecDecodedLmsw(pVCpu, pVmxTransient->cbInstr,
12075 VMX_EXIT_QUAL_CRX_LMSW_DATA(uExitQualification));
12076 AssertMsg( rcStrict == VINF_SUCCESS
12077 || rcStrict == VINF_IEM_RAISED_XCPT
12078 || rcStrict == VINF_PGM_CHANGE_MODE,
12079 ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12080
12081 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_CR0);
12082 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitLmsw);
12083 Log4(("CRX LMSW rcStrict=%d\n", VBOXSTRICTRC_VAL(rcStrict)));
12084 break;
12085 }
12086
12087 default:
12088 AssertMsgFailedReturn(("Invalid access-type in Mov CRx VM-exit qualification %#x\n", uAccessType),
12089 VERR_VMX_UNEXPECTED_EXCEPTION);
12090 }
12091
12092 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, rcStrict != VINF_IEM_RAISED_XCPT ? HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS
12093 : HM_CHANGED_XCPT_RAISED_MASK);
12094 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitMovCRx, y2);
12095 NOREF(pVM);
12096 return rcStrict;
12097}
12098
12099
12100/**
12101 * VM-exit handler for I/O instructions (VMX_EXIT_IO_INSTR). Conditional
12102 * VM-exit.
12103 */
12104HMVMX_EXIT_DECL hmR0VmxExitIoInstr(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12105{
12106 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12107 STAM_PROFILE_ADV_START(&pVCpu->hm.s.StatExitIO, y1);
12108 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12109
12110 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12111 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12112 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK
12113 | CPUMCTX_EXTRN_SREG_MASK
12114 | CPUMCTX_EXTRN_EFER);
12115 /* EFER also required for longmode checks in EMInterpretDisasCurrent(), but it's always up-to-date. */
12116 AssertRCReturn(rc, rc);
12117
12118 /* Refer Intel spec. 27-5. "Exit Qualifications for I/O Instructions" for the format. */
12119 uint32_t uIOPort = VMX_EXIT_QUAL_IO_PORT(pVmxTransient->uExitQualification);
12120 uint8_t uIOWidth = VMX_EXIT_QUAL_IO_WIDTH(pVmxTransient->uExitQualification);
12121 bool fIOWrite = ( VMX_EXIT_QUAL_IO_DIRECTION(pVmxTransient->uExitQualification)
12122 == VMX_EXIT_QUAL_IO_DIRECTION_OUT);
12123 bool fIOString = VMX_EXIT_QUAL_IO_IS_STRING(pVmxTransient->uExitQualification);
12124 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
12125 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12126 AssertReturn(uIOWidth <= 3 && uIOWidth != 2, VERR_VMX_IPE_1);
12127
12128 /*
12129 * Update exit history to see if this exit can be optimized.
12130 */
12131 VBOXSTRICTRC rcStrict;
12132 PCEMEXITREC pExitRec = NULL;
12133 if ( !fGstStepping
12134 && !fDbgStepping)
12135 pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12136 !fIOString
12137 ? !fIOWrite
12138 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_READ)
12139 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_WRITE)
12140 : !fIOWrite
12141 ? EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_READ)
12142 : EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_IO_PORT_STR_WRITE),
12143 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12144 if (!pExitRec)
12145 {
12146 /* I/O operation lookup arrays. */
12147 static uint32_t const s_aIOSizes[4] = { 1, 2, 0, 4 }; /* Size of the I/O accesses. */
12148 static uint32_t const s_aIOOpAnd[4] = { 0xff, 0xffff, 0, 0xffffffff }; /* AND masks for saving result in AL/AX/EAX. */
12149 uint32_t const cbValue = s_aIOSizes[uIOWidth];
12150 uint32_t const cbInstr = pVmxTransient->cbInstr;
12151 bool fUpdateRipAlready = false; /* ugly hack, should be temporary. */
12152 PVM pVM = pVCpu->CTX_SUFF(pVM);
12153 if (fIOString)
12154 {
12155 /*
12156 * INS/OUTS - I/O String instruction.
12157 *
12158 * Use instruction-information if available, otherwise fall back on
12159 * interpreting the instruction.
12160 */
12161 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c str\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12162 fIOWrite ? 'w' : 'r'));
12163 AssertReturn(pMixedCtx->dx == uIOPort, VERR_VMX_IPE_2);
12164 if (MSR_IA32_VMX_BASIC_INFO_VMCS_INS_OUTS(pVM->hm.s.vmx.Msrs.u64BasicInfo))
12165 {
12166 int rc2 = hmR0VmxReadExitInstrInfoVmcs(pVmxTransient);
12167 AssertRCReturn(rc2, rc2);
12168 AssertReturn(pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize <= 2, VERR_VMX_IPE_3);
12169 AssertCompile(IEMMODE_16BIT == 0 && IEMMODE_32BIT == 1 && IEMMODE_64BIT == 2);
12170 IEMMODE const enmAddrMode = (IEMMODE)pVmxTransient->ExitInstrInfo.StrIo.u3AddrSize;
12171 bool const fRep = VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification);
12172 if (fIOWrite)
12173 rcStrict = IEMExecStringIoWrite(pVCpu, cbValue, enmAddrMode, fRep, cbInstr,
12174 pVmxTransient->ExitInstrInfo.StrIo.iSegReg, true /*fIoChecked*/);
12175 else
12176 {
12177 /*
12178 * The segment prefix for INS cannot be overridden and is always ES. We can safely assume X86_SREG_ES.
12179 * Hence "iSegReg" field is undefined in the instruction-information field in VT-x for INS.
12180 * See Intel Instruction spec. for "INS".
12181 * See Intel spec. Table 27-8 "Format of the VM-Exit Instruction-Information Field as Used for INS and OUTS".
12182 */
12183 rcStrict = IEMExecStringIoRead(pVCpu, cbValue, enmAddrMode, fRep, cbInstr, true /*fIoChecked*/);
12184 }
12185 }
12186 else
12187 rcStrict = IEMExecOne(pVCpu);
12188
12189 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12190 fUpdateRipAlready = true;
12191 }
12192 else
12193 {
12194 /*
12195 * IN/OUT - I/O instruction.
12196 */
12197 Log4(("CS:RIP=%04x:%08RX64 %#06x/%u %c\n", pMixedCtx->cs.Sel, pMixedCtx->rip, uIOPort, cbValue,
12198 fIOWrite ? 'w' : 'r'));
12199 uint32_t const uAndVal = s_aIOOpAnd[uIOWidth];
12200 Assert(!VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification));
12201 if (fIOWrite)
12202 {
12203 rcStrict = IOMIOPortWrite(pVM, pVCpu, uIOPort, pMixedCtx->eax & uAndVal, cbValue);
12204 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIOWrite);
12205 }
12206 else
12207 {
12208 uint32_t u32Result = 0;
12209 rcStrict = IOMIOPortRead(pVM, pVCpu, uIOPort, &u32Result, cbValue);
12210 if (IOM_SUCCESS(rcStrict))
12211 {
12212 /* Save result of I/O IN instr. in AL/AX/EAX. */
12213 pMixedCtx->eax = (pMixedCtx->eax & ~uAndVal) | (u32Result & uAndVal);
12214 }
12215 else if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12216 HMR0SavePendingIOPortRead(pVCpu, pMixedCtx->rip, pMixedCtx->rip + cbInstr, uIOPort, uAndVal, cbValue);
12217 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIORead);
12218 }
12219 }
12220
12221 if (IOM_SUCCESS(rcStrict))
12222 {
12223 if (!fUpdateRipAlready)
12224 {
12225 hmR0VmxAdvanceGuestRipBy(pVCpu, pMixedCtx, cbInstr);
12226 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP);
12227 }
12228
12229 /*
12230 * INS/OUTS with REP prefix updates RFLAGS, can be observed with triple-fault guru
12231 * while booting Fedora 17 64-bit guest.
12232 *
12233 * See Intel Instruction reference for REP/REPE/REPZ/REPNE/REPNZ.
12234 */
12235 if (fIOString)
12236 {
12237 /** @todo Single-step for INS/OUTS with REP prefix? */
12238 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
12239 }
12240 else if ( !fDbgStepping
12241 && fGstStepping)
12242 {
12243 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12244 AssertRCReturn(rc, rc);
12245 }
12246
12247 /*
12248 * If any I/O breakpoints are armed, we need to check if one triggered
12249 * and take appropriate action.
12250 * Note that the I/O breakpoint type is undefined if CR4.DE is 0.
12251 */
12252 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12253 AssertRCReturn(rc, rc);
12254
12255 /** @todo Optimize away the DBGFBpIsHwIoArmed call by having DBGF tell the
12256 * execution engines about whether hyper BPs and such are pending. */
12257 uint32_t const uDr7 = pMixedCtx->dr[7];
12258 if (RT_UNLIKELY( ( (uDr7 & X86_DR7_ENABLED_MASK)
12259 && X86_DR7_ANY_RW_IO(uDr7)
12260 && (pMixedCtx->cr4 & X86_CR4_DE))
12261 || DBGFBpIsHwIoArmed(pVM)))
12262 {
12263 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxIoCheck);
12264
12265 /* We're playing with the host CPU state here, make sure we don't preempt or longjmp. */
12266 VMMRZCallRing3Disable(pVCpu);
12267 HM_DISABLE_PREEMPT();
12268
12269 bool fIsGuestDbgActive = CPUMR0DebugStateMaybeSaveGuest(pVCpu, true /* fDr6 */);
12270
12271 VBOXSTRICTRC rcStrict2 = DBGFBpCheckIo(pVM, pVCpu, pMixedCtx, uIOPort, cbValue);
12272 if (rcStrict2 == VINF_EM_RAW_GUEST_TRAP)
12273 {
12274 /* Raise #DB. */
12275 if (fIsGuestDbgActive)
12276 ASMSetDR6(pMixedCtx->dr[6]);
12277 if (pMixedCtx->dr[7] != uDr7)
12278 pVCpu->hm.s.fCtxChanged |= HM_CHANGED_GUEST_DR7;
12279
12280 hmR0VmxSetPendingXcptDB(pVCpu, pMixedCtx);
12281 }
12282 /* rcStrict is VINF_SUCCESS, VINF_IOM_R3_IOPORT_COMMIT_WRITE, or in [VINF_EM_FIRST..VINF_EM_LAST],
12283 however we can ditch VINF_IOM_R3_IOPORT_COMMIT_WRITE as it has VMCPU_FF_IOM as backup. */
12284 else if ( rcStrict2 != VINF_SUCCESS
12285 && (rcStrict == VINF_SUCCESS || rcStrict2 < rcStrict))
12286 rcStrict = rcStrict2;
12287 AssertCompile(VINF_EM_LAST < VINF_IOM_R3_IOPORT_COMMIT_WRITE);
12288
12289 HM_RESTORE_PREEMPT();
12290 VMMRZCallRing3Enable(pVCpu);
12291 }
12292 }
12293
12294#ifdef VBOX_STRICT
12295 if (rcStrict == VINF_IOM_R3_IOPORT_READ)
12296 Assert(!fIOWrite);
12297 else if (rcStrict == VINF_IOM_R3_IOPORT_WRITE || rcStrict == VINF_IOM_R3_IOPORT_COMMIT_WRITE)
12298 Assert(fIOWrite);
12299 else
12300 {
12301# if 0 /** @todo r=bird: This is missing a bunch of VINF_EM_FIRST..VINF_EM_LAST
12302 * statuses, that the VMM device and some others may return. See
12303 * IOM_SUCCESS() for guidance. */
12304 AssertMsg( RT_FAILURE(rcStrict)
12305 || rcStrict == VINF_SUCCESS
12306 || rcStrict == VINF_EM_RAW_EMULATE_INSTR
12307 || rcStrict == VINF_EM_DBG_BREAKPOINT
12308 || rcStrict == VINF_EM_RAW_GUEST_TRAP
12309 || rcStrict == VINF_EM_RAW_TO_R3
12310 || rcStrict == VINF_TRPM_XCPT_DISPATCHED, ("%Rrc\n", VBOXSTRICTRC_VAL(rcStrict)));
12311# endif
12312 }
12313#endif
12314 STAM_PROFILE_ADV_STOP(&pVCpu->hm.s.StatExitIO, y1);
12315 }
12316 else
12317 {
12318 /*
12319 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12320 */
12321 int rc2 = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12322 AssertRCReturn(rc2, rc2);
12323 STAM_COUNTER_INC(!fIOString ? fIOWrite ? &pVCpu->hm.s.StatExitIOWrite : &pVCpu->hm.s.StatExitIORead
12324 : fIOWrite ? &pVCpu->hm.s.StatExitIOStringWrite : &pVCpu->hm.s.StatExitIOStringRead);
12325 Log4(("IOExit/%u: %04x:%08RX64: %s%s%s %#x LB %u -> EMHistoryExec\n",
12326 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12327 VMX_EXIT_QUAL_IO_IS_REP(pVmxTransient->uExitQualification) ? "REP " : "",
12328 fIOWrite ? "OUT" : "IN", fIOString ? "S" : "", uIOPort, uIOWidth));
12329
12330 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12331 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12332
12333 Log4(("IOExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12334 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12335 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12336 }
12337 return rcStrict;
12338}
12339
12340
12341/**
12342 * VM-exit handler for task switches (VMX_EXIT_TASK_SWITCH). Unconditional
12343 * VM-exit.
12344 */
12345HMVMX_EXIT_DECL hmR0VmxExitTaskSwitch(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12346{
12347 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12348
12349 /* Check if this task-switch occurred while delivery an event through the guest IDT. */
12350 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12351 AssertRCReturn(rc, rc);
12352 if (VMX_EXIT_QUAL_TASK_SWITCH_TYPE(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_TASK_SWITCH_TYPE_IDT)
12353 {
12354 rc = hmR0VmxReadIdtVectoringInfoVmcs(pVmxTransient);
12355 AssertRCReturn(rc, rc);
12356 if (VMX_IDT_VECTORING_INFO_VALID(pVmxTransient->uIdtVectoringInfo))
12357 {
12358 uint32_t uErrCode;
12359 RTGCUINTPTR GCPtrFaultAddress;
12360 uint32_t const uIntType = VMX_IDT_VECTORING_INFO_TYPE(pVmxTransient->uIdtVectoringInfo);
12361 uint32_t const uVector = VMX_IDT_VECTORING_INFO_VECTOR(pVmxTransient->uIdtVectoringInfo);
12362 bool const fErrorCodeValid = VMX_IDT_VECTORING_INFO_ERROR_CODE_IS_VALID(pVmxTransient->uIdtVectoringInfo);
12363 if (fErrorCodeValid)
12364 {
12365 rc = hmR0VmxReadIdtVectoringErrorCodeVmcs(pVmxTransient);
12366 AssertRCReturn(rc, rc);
12367 uErrCode = pVmxTransient->uIdtVectoringErrorCode;
12368 }
12369 else
12370 uErrCode = 0;
12371
12372 if ( uIntType == VMX_IDT_VECTORING_INFO_TYPE_HW_XCPT
12373 && uVector == X86_XCPT_PF)
12374 GCPtrFaultAddress = pMixedCtx->cr2;
12375 else
12376 GCPtrFaultAddress = 0;
12377
12378 hmR0VmxSetPendingEvent(pVCpu, VMX_ENTRY_INT_INFO_FROM_EXIT_IDT_INFO(pVmxTransient->uIdtVectoringInfo),
12379 0 /* cbInstr */, uErrCode, GCPtrFaultAddress);
12380
12381 Log4(("Pending event on TaskSwitch uIntType=%#x uVector=%#x\n", uIntType, uVector));
12382 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12383 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12384 }
12385 }
12386
12387 /* Fall back to the interpreter to emulate the task-switch. */
12388 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitTaskSwitch);
12389 return VERR_EM_INTERPRETER;
12390}
12391
12392
12393/**
12394 * VM-exit handler for monitor-trap-flag (VMX_EXIT_MTF). Conditional VM-exit.
12395 */
12396HMVMX_EXIT_DECL hmR0VmxExitMtf(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12397{
12398 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12399 Assert(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG);
12400 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MONITOR_TRAP_FLAG;
12401 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12402 AssertRCReturn(rc, rc);
12403 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitMtf);
12404 return VINF_EM_DBG_STEPPED;
12405}
12406
12407
12408/**
12409 * VM-exit handler for APIC access (VMX_EXIT_APIC_ACCESS). Conditional VM-exit.
12410 */
12411HMVMX_EXIT_DECL hmR0VmxExitApicAccess(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12412{
12413 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12414
12415 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitApicAccess);
12416
12417 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12418 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12419 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12420 {
12421 /* For some crazy guest, if an event delivery causes an APIC-access VM-exit, go to instruction emulation. */
12422 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12423 {
12424 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12425 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12426 }
12427 }
12428 else
12429 {
12430 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12431 rcStrict1 = VINF_SUCCESS;
12432 return rcStrict1;
12433 }
12434
12435 /* IOMMIOPhysHandler() below may call into IEM, save the necessary state. */
12436 int rc = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12437 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12438 AssertRCReturn(rc, rc);
12439
12440 /* See Intel spec. 27-6 "Exit Qualifications for APIC-access VM-exits from Linear Accesses & Guest-Phyiscal Addresses" */
12441 uint32_t uAccessType = VMX_EXIT_QUAL_APIC_ACCESS_TYPE(pVmxTransient->uExitQualification);
12442 VBOXSTRICTRC rcStrict2;
12443 switch (uAccessType)
12444 {
12445 case VMX_APIC_ACCESS_TYPE_LINEAR_WRITE:
12446 case VMX_APIC_ACCESS_TYPE_LINEAR_READ:
12447 {
12448 AssertMsg( !(pVCpu->hm.s.vmx.u32ProcCtls & VMX_VMCS_CTRL_PROC_EXEC_USE_TPR_SHADOW)
12449 || VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification) != XAPIC_OFF_TPR,
12450 ("hmR0VmxExitApicAccess: can't access TPR offset while using TPR shadowing.\n"));
12451
12452 RTGCPHYS GCPhys = pVCpu->hm.s.vmx.u64MsrApicBase; /* Always up-to-date, u64MsrApicBase is not part of the VMCS. */
12453 GCPhys &= PAGE_BASE_GC_MASK;
12454 GCPhys += VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification);
12455 PVM pVM = pVCpu->CTX_SUFF(pVM);
12456 Log4Func(("Linear access uAccessType=%#x GCPhys=%#RGp Off=%#x\n", uAccessType, GCPhys,
12457 VMX_EXIT_QUAL_APIC_ACCESS_OFFSET(pVmxTransient->uExitQualification)));
12458
12459 rcStrict2 = IOMMMIOPhysHandler(pVM, pVCpu,
12460 uAccessType == VMX_APIC_ACCESS_TYPE_LINEAR_READ ? 0 : X86_TRAP_PF_RW,
12461 CPUMCTX2CORE(pMixedCtx), GCPhys);
12462 Log4Func(("IOMMMIOPhysHandler returned %Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12463 if ( rcStrict2 == VINF_SUCCESS
12464 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12465 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12466 {
12467 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12468 | HM_CHANGED_GUEST_RSP
12469 | HM_CHANGED_GUEST_RFLAGS
12470 | HM_CHANGED_GUEST_APIC_TPR);
12471 rcStrict2 = VINF_SUCCESS;
12472 }
12473 break;
12474 }
12475
12476 default:
12477 Log4Func(("uAccessType=%#x\n", uAccessType));
12478 rcStrict2 = VINF_EM_RAW_EMULATE_INSTR;
12479 break;
12480 }
12481
12482 if (rcStrict2 != VINF_SUCCESS)
12483 STAM_COUNTER_INC(&pVCpu->hm.s.StatSwitchApicAccessToR3);
12484 return rcStrict2;
12485}
12486
12487
12488/**
12489 * VM-exit handler for debug-register accesses (VMX_EXIT_MOV_DRX). Conditional
12490 * VM-exit.
12491 */
12492HMVMX_EXIT_DECL hmR0VmxExitMovDRx(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12493{
12494 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12495
12496 /* We should -not- get this VM-exit if the guest's debug registers were active. */
12497 if (pVmxTransient->fWasGuestDebugStateActive)
12498 {
12499 AssertMsgFailed(("Unexpected MOV DRx exit. pVCpu=%p pMixedCtx=%p\n", pVCpu, pMixedCtx));
12500 HMVMX_RETURN_UNEXPECTED_EXIT();
12501 }
12502
12503 if ( !pVCpu->hm.s.fSingleInstruction
12504 && !pVmxTransient->fWasHyperDebugStateActive)
12505 {
12506 Assert(!DBGFIsStepping(pVCpu));
12507 Assert(pVCpu->hm.s.vmx.u32XcptBitmap & RT_BIT_32(X86_XCPT_DB));
12508
12509 /* Don't intercept MOV DRx any more. */
12510 pVCpu->hm.s.vmx.u32ProcCtls &= ~VMX_VMCS_CTRL_PROC_EXEC_MOV_DR_EXIT;
12511 int rc = VMXWriteVmcs32(VMX_VMCS32_CTRL_PROC_EXEC, pVCpu->hm.s.vmx.u32ProcCtls);
12512 AssertRCReturn(rc, rc);
12513
12514 /* We're playing with the host CPU state here, make sure we can't preempt or longjmp. */
12515 VMMRZCallRing3Disable(pVCpu);
12516 HM_DISABLE_PREEMPT();
12517
12518 /* Save the host & load the guest debug state, restart execution of the MOV DRx instruction. */
12519 CPUMR0LoadGuestDebugState(pVCpu, true /* include DR6 */);
12520 Assert(CPUMIsGuestDebugStateActive(pVCpu) || HC_ARCH_BITS == 32);
12521
12522 HM_RESTORE_PREEMPT();
12523 VMMRZCallRing3Enable(pVCpu);
12524
12525#ifdef VBOX_WITH_STATISTICS
12526 rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12527 AssertRCReturn(rc, rc);
12528 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12529 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12530 else
12531 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12532#endif
12533 STAM_COUNTER_INC(&pVCpu->hm.s.StatDRxContextSwitch);
12534 return VINF_SUCCESS;
12535 }
12536
12537 /*
12538 * EMInterpretDRx[Write|Read]() calls CPUMIsGuestIn64BitCode() which requires EFER, CS. EFER is always up-to-date.
12539 * Update the segment registers and DR7 from the CPU.
12540 */
12541 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12542 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_SREG_MASK
12543 | CPUMCTX_EXTRN_DR7);
12544 AssertRCReturn(rc, rc);
12545 Log4Func(("CS:RIP=%04x:%08RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
12546
12547 PVM pVM = pVCpu->CTX_SUFF(pVM);
12548 if (VMX_EXIT_QUAL_DRX_DIRECTION(pVmxTransient->uExitQualification) == VMX_EXIT_QUAL_DRX_DIRECTION_WRITE)
12549 {
12550 rc = EMInterpretDRxWrite(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12551 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification),
12552 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification));
12553 if (RT_SUCCESS(rc))
12554 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_DR7);
12555 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxWrite);
12556 }
12557 else
12558 {
12559 rc = EMInterpretDRxRead(pVM, pVCpu, CPUMCTX2CORE(pMixedCtx),
12560 VMX_EXIT_QUAL_DRX_GENREG(pVmxTransient->uExitQualification),
12561 VMX_EXIT_QUAL_DRX_REGISTER(pVmxTransient->uExitQualification));
12562 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitDRxRead);
12563 }
12564
12565 Assert(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER);
12566 if (RT_SUCCESS(rc))
12567 {
12568 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12569 AssertRCReturn(rc2, rc2);
12570 return VINF_SUCCESS;
12571 }
12572 return rc;
12573}
12574
12575
12576/**
12577 * VM-exit handler for EPT misconfiguration (VMX_EXIT_EPT_MISCONFIG).
12578 * Conditional VM-exit.
12579 */
12580HMVMX_EXIT_DECL hmR0VmxExitEptMisconfig(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12581{
12582 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12583 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12584
12585 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12586 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12587 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12588 {
12589 /* If event delivery causes an EPT misconfig (MMIO), go back to instruction emulation as otherwise
12590 injecting the original pending event would most likely cause the same EPT misconfig VM-exit. */
12591 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12592 {
12593 STAM_COUNTER_INC(&pVCpu->hm.s.StatInjectPendingInterpret);
12594 return VINF_EM_RAW_INJECT_TRPM_EVENT;
12595 }
12596 }
12597 else
12598 {
12599 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12600 rcStrict1 = VINF_SUCCESS;
12601 return rcStrict1;
12602 }
12603
12604 /*
12605 * Get sufficent state and update the exit history entry.
12606 */
12607 RTGCPHYS GCPhys;
12608 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12609 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12610 AssertRCReturn(rc, rc);
12611
12612 VBOXSTRICTRC rcStrict;
12613 PCEMEXITREC pExitRec = EMHistoryUpdateFlagsAndTypeAndPC(pVCpu,
12614 EMEXIT_MAKE_FT(EMEXIT_F_KIND_EM | EMEXIT_F_HM, EMEXITTYPE_MMIO),
12615 pVCpu->cpum.GstCtx.rip + pVCpu->cpum.GstCtx.cs.u64Base);
12616 if (!pExitRec)
12617 {
12618 /*
12619 * If we succeed, resume guest execution.
12620 * If we fail in interpreting the instruction because we couldn't get the guest physical address
12621 * of the page containing the instruction via the guest's page tables (we would invalidate the guest page
12622 * in the host TLB), resume execution which would cause a guest page fault to let the guest handle this
12623 * weird case. See @bugref{6043}.
12624 */
12625 PVM pVM = pVCpu->CTX_SUFF(pVM);
12626 rcStrict = PGMR0Trap0eHandlerNPMisconfig(pVM, pVCpu, PGMMODE_EPT, CPUMCTX2CORE(pMixedCtx), GCPhys, UINT32_MAX);
12627 Log4(("EPT misconfig at %#RGp RIP=%#RX64 rc=%Rrc\n", GCPhys, pMixedCtx->rip, VBOXSTRICTRC_VAL(rcStrict)));
12628 if ( rcStrict == VINF_SUCCESS
12629 || rcStrict == VERR_PAGE_TABLE_NOT_PRESENT
12630 || rcStrict == VERR_PAGE_NOT_PRESENT)
12631 {
12632 /* Successfully handled MMIO operation. */
12633 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12634 | HM_CHANGED_GUEST_RSP
12635 | HM_CHANGED_GUEST_RFLAGS
12636 | HM_CHANGED_GUEST_APIC_TPR);
12637 rcStrict = VINF_SUCCESS;
12638 }
12639 }
12640 else
12641 {
12642 /*
12643 * Frequent exit or something needing probing. Get state and call EMHistoryExec.
12644 */
12645 Assert(pMixedCtx == &pVCpu->cpum.GstCtx);
12646 int rc2 = hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12647 AssertRCReturn(rc2, rc2);
12648
12649 Log4(("EptMisscfgExit/%u: %04x:%08RX64: %RGp -> EMHistoryExec\n",
12650 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip, GCPhys));
12651
12652 rcStrict = EMHistoryExec(pVCpu, pExitRec, 0);
12653 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
12654
12655 Log4(("EptMisscfgExit/%u: %04x:%08RX64: EMHistoryExec -> %Rrc + %04x:%08RX64\n",
12656 pVCpu->idCpu, pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip,
12657 VBOXSTRICTRC_VAL(rcStrict), pVCpu->cpum.GstCtx.cs.Sel, pVCpu->cpum.GstCtx.rip));
12658 }
12659 return VBOXSTRICTRC_TODO(rcStrict);
12660}
12661
12662
12663/**
12664 * VM-exit handler for EPT violation (VMX_EXIT_EPT_VIOLATION). Conditional
12665 * VM-exit.
12666 */
12667HMVMX_EXIT_DECL hmR0VmxExitEptViolation(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12668{
12669 HMVMX_VALIDATE_EXIT_HANDLER_PARAMS();
12670 Assert(pVCpu->CTX_SUFF(pVM)->hm.s.fNestedPaging);
12671
12672 /* If this VM-exit occurred while delivering an event through the guest IDT, handle it accordingly. */
12673 VBOXSTRICTRC rcStrict1 = hmR0VmxCheckExitDueToEventDelivery(pVCpu, pMixedCtx, pVmxTransient);
12674 if (RT_LIKELY(rcStrict1 == VINF_SUCCESS))
12675 {
12676 /* In the unlikely case that the EPT violation happened as a result of delivering an event, log it. */
12677 if (RT_UNLIKELY(pVCpu->hm.s.Event.fPending))
12678 Log4Func(("EPT violation with an event pending u64IntInfo=%#RX64\n", pVCpu->hm.s.Event.u64IntInfo));
12679 }
12680 else
12681 {
12682 if (rcStrict1 == VINF_HM_DOUBLE_FAULT)
12683 rcStrict1 = VINF_SUCCESS;
12684 return rcStrict1;
12685 }
12686
12687 RTGCPHYS GCPhys;
12688 int rc = VMXReadVmcs64(VMX_VMCS64_EXIT_GUEST_PHYS_ADDR_FULL, &GCPhys);
12689 rc |= hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12690 rc |= hmR0VmxImportGuestState(pVCpu, IEM_CPUMCTX_EXTRN_MUST_MASK);
12691 AssertRCReturn(rc, rc);
12692
12693 /* Intel spec. Table 27-7 "Exit Qualifications for EPT violations". */
12694 AssertMsg(((pVmxTransient->uExitQualification >> 7) & 3) != 2, ("%#RX64", pVmxTransient->uExitQualification));
12695
12696 RTGCUINT uErrorCode = 0;
12697 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_INSTR_FETCH)
12698 uErrorCode |= X86_TRAP_PF_ID;
12699 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_DATA_WRITE)
12700 uErrorCode |= X86_TRAP_PF_RW;
12701 if (pVmxTransient->uExitQualification & VMX_EXIT_QUAL_EPT_ENTRY_PRESENT)
12702 uErrorCode |= X86_TRAP_PF_P;
12703
12704 TRPMAssertXcptPF(pVCpu, GCPhys, uErrorCode);
12705
12706 Log4Func(("EPT violation %#x at %#RX64 ErrorCode %#x CS:RIP=%04x:%08RX64\n", pVmxTransient->uExitQualification, GCPhys,
12707 uErrorCode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12708
12709 /* Handle the pagefault trap for the nested shadow table. */
12710 PVM pVM = pVCpu->CTX_SUFF(pVM);
12711 VBOXSTRICTRC rcStrict2 = PGMR0Trap0eHandlerNestedPaging(pVM, pVCpu, PGMMODE_EPT, uErrorCode, CPUMCTX2CORE(pMixedCtx), GCPhys);
12712 TRPMResetTrap(pVCpu);
12713
12714 /* Same case as PGMR0Trap0eHandlerNPMisconfig(). See comment above, @bugref{6043}. */
12715 if ( rcStrict2 == VINF_SUCCESS
12716 || rcStrict2 == VERR_PAGE_TABLE_NOT_PRESENT
12717 || rcStrict2 == VERR_PAGE_NOT_PRESENT)
12718 {
12719 /* Successfully synced our nested page tables. */
12720 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitReasonNpf);
12721 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
12722 | HM_CHANGED_GUEST_RSP
12723 | HM_CHANGED_GUEST_RFLAGS);
12724 return VINF_SUCCESS;
12725 }
12726
12727 Log4Func(("EPT return to ring-3 rcStrict2=%Rrc\n", VBOXSTRICTRC_VAL(rcStrict2)));
12728 return rcStrict2;
12729}
12730
12731/** @} */
12732
12733/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12734/* -=-=-=-=-=-=-=-=-=- VM-exit Exception Handlers -=-=-=-=-=-=-=-=-=-=- */
12735/* -=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=--=-=-=-=-=-=-=-=-=-=-=-=-=-= */
12736
12737/** @name VM-exit exception handlers.
12738 * @{
12739 */
12740
12741/**
12742 * VM-exit exception handler for \#MF (Math Fault: floating point exception).
12743 */
12744static int hmR0VmxExitXcptMF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12745{
12746 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12747 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestMF);
12748
12749 int rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CR0);
12750 AssertRCReturn(rc, rc);
12751
12752 if (!(pMixedCtx->cr0 & X86_CR0_NE))
12753 {
12754 /* Convert a #MF into a FERR -> IRQ 13. See @bugref{6117}. */
12755 rc = PDMIsaSetIrq(pVCpu->CTX_SUFF(pVM), 13, 1, 0 /* uTagSrc */);
12756
12757 /** @todo r=ramshankar: The Intel spec. does -not- specify that this VM-exit
12758 * provides VM-exit instruction length. If this causes problem later,
12759 * disassemble the instruction like it's done on AMD-V. */
12760 int rc2 = hmR0VmxAdvanceGuestRip(pVCpu, pMixedCtx, pVmxTransient);
12761 AssertRCReturn(rc2, rc2);
12762 return rc;
12763 }
12764
12765 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12766 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12767 return rc;
12768}
12769
12770
12771/**
12772 * VM-exit exception handler for \#BP (Breakpoint exception).
12773 */
12774static int hmR0VmxExitXcptBP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12775{
12776 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12777 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestBP);
12778
12779 int rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12780 AssertRCReturn(rc, rc);
12781
12782 rc = DBGFRZTrap03Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx));
12783 if (rc == VINF_EM_RAW_GUEST_TRAP)
12784 {
12785 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12786 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12787 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12788 AssertRCReturn(rc, rc);
12789
12790 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12791 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12792 }
12793
12794 Assert(rc == VINF_SUCCESS || rc == VINF_EM_RAW_GUEST_TRAP || rc == VINF_EM_DBG_BREAKPOINT);
12795 return rc;
12796}
12797
12798
12799/**
12800 * VM-exit exception handler for \#AC (alignment check exception).
12801 */
12802static int hmR0VmxExitXcptAC(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12803{
12804 RT_NOREF_PV(pMixedCtx);
12805 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12806
12807 /*
12808 * Re-inject it. We'll detect any nesting before getting here.
12809 */
12810 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12811 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12812 AssertRCReturn(rc, rc);
12813 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
12814
12815 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12816 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12817 return VINF_SUCCESS;
12818}
12819
12820
12821/**
12822 * VM-exit exception handler for \#DB (Debug exception).
12823 */
12824static int hmR0VmxExitXcptDB(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12825{
12826 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12827 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestDB);
12828
12829 /*
12830 * Get the DR6-like values from the VM-exit qualification and pass it to DBGF
12831 * for processing.
12832 */
12833 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
12834
12835 /* Refer Intel spec. Table 27-1. "Exit Qualifications for debug exceptions" for the format. */
12836 uint64_t uDR6 = X86_DR6_INIT_VAL;
12837 uDR6 |= ( pVmxTransient->uExitQualification
12838 & (X86_DR6_B0 | X86_DR6_B1 | X86_DR6_B2 | X86_DR6_B3 | X86_DR6_BD | X86_DR6_BS));
12839
12840 rc = DBGFRZTrap01Handler(pVCpu->CTX_SUFF(pVM), pVCpu, CPUMCTX2CORE(pMixedCtx), uDR6, pVCpu->hm.s.fSingleInstruction);
12841 Log6Func(("rc=%Rrc\n", rc));
12842 if (rc == VINF_EM_RAW_GUEST_TRAP)
12843 {
12844 /*
12845 * The exception was for the guest. Update DR6, DR7.GD and
12846 * IA32_DEBUGCTL.LBR before forwarding it.
12847 * (See Intel spec. 27.1 "Architectural State before a VM-Exit".)
12848 */
12849 VMMRZCallRing3Disable(pVCpu);
12850 HM_DISABLE_PREEMPT();
12851
12852 pMixedCtx->dr[6] &= ~X86_DR6_B_MASK;
12853 pMixedCtx->dr[6] |= uDR6;
12854 if (CPUMIsGuestDebugStateActive(pVCpu))
12855 ASMSetDR6(pMixedCtx->dr[6]);
12856
12857 HM_RESTORE_PREEMPT();
12858 VMMRZCallRing3Enable(pVCpu);
12859
12860 rc = hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_DR7);
12861 AssertRCReturn(rc, rc);
12862
12863 /* X86_DR7_GD will be cleared if DRx accesses should be trapped inside the guest. */
12864 pMixedCtx->dr[7] &= ~X86_DR7_GD;
12865
12866 /* Paranoia. */
12867 pMixedCtx->dr[7] &= ~X86_DR7_RAZ_MASK;
12868 pMixedCtx->dr[7] |= X86_DR7_RA1_MASK;
12869
12870 rc = VMXWriteVmcs32(VMX_VMCS_GUEST_DR7, (uint32_t)pMixedCtx->dr[7]);
12871 AssertRCReturn(rc, rc);
12872
12873 /*
12874 * Raise #DB in the guest.
12875 *
12876 * It is important to reflect exactly what the VM-exit gave us (preserving the
12877 * interruption-type) rather than use hmR0VmxSetPendingXcptDB() as the #DB could've
12878 * been raised while executing ICEBP (INT1) and not the regular #DB. Thus it may
12879 * trigger different handling in the CPU (like skipping DPL checks), see @bugref{6398}.
12880 *
12881 * Intel re-documented ICEBP/INT1 on May 2018 previously documented as part of
12882 * Intel 386, see Intel spec. 24.8.3 "VM-Entry Controls for Event Injection".
12883 */
12884 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12885 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12886 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12887 AssertRCReturn(rc, rc);
12888 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12889 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12890 return VINF_SUCCESS;
12891 }
12892
12893 /*
12894 * Not a guest trap, must be a hypervisor related debug event then.
12895 * Update DR6 in case someone is interested in it.
12896 */
12897 AssertMsg(rc == VINF_EM_DBG_STEPPED || rc == VINF_EM_DBG_BREAKPOINT, ("%Rrc\n", rc));
12898 AssertReturn(pVmxTransient->fWasHyperDebugStateActive, VERR_HM_IPE_5);
12899 CPUMSetHyperDR6(pVCpu, uDR6);
12900
12901 return rc;
12902}
12903
12904/**
12905 * VM-exit exception handler for \#GP (General-protection exception).
12906 *
12907 * @remarks Requires pVmxTransient->uExitIntInfo to be up-to-date.
12908 */
12909static int hmR0VmxExitXcptGP(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
12910{
12911 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
12912 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestGP);
12913
12914 int rc;
12915 if (pVCpu->hm.s.vmx.RealMode.fRealOnV86Active)
12916 { /* likely */ }
12917 else
12918 {
12919#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
12920 Assert(pVCpu->hm.s.fUsingDebugLoop);
12921#endif
12922 /* If the guest is not in real-mode or we have unrestricted execution support, reflect #GP to the guest. */
12923 rc = hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
12924 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
12925 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
12926 rc |= hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12927 AssertRCReturn(rc, rc);
12928 Log4Func(("Gst: CS:RIP %04x:%08RX64 ErrorCode=%#x CR0=%#RX64 CPL=%u TR=%#04x\n", pMixedCtx->cs.Sel, pMixedCtx->rip,
12929 pVmxTransient->uExitIntErrorCode, pMixedCtx->cr0, CPUMGetGuestCPL(pVCpu), pMixedCtx->tr.Sel));
12930 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
12931 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
12932 return rc;
12933 }
12934
12935 Assert(CPUMIsGuestInRealModeEx(pMixedCtx));
12936 Assert(!pVCpu->CTX_SUFF(pVM)->hm.s.vmx.fUnrestrictedGuest);
12937
12938 /* EMInterpretDisasCurrent() requires a lot of the state, save the entire state. */
12939 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
12940 AssertRCReturn(rc, rc);
12941
12942 PDISCPUSTATE pDis = &pVCpu->hm.s.DisState;
12943 uint32_t cbOp = 0;
12944 PVM pVM = pVCpu->CTX_SUFF(pVM);
12945 bool fDbgStepping = pVCpu->hm.s.fSingleInstruction;
12946 rc = EMInterpretDisasCurrent(pVM, pVCpu, pDis, &cbOp);
12947 if (RT_SUCCESS(rc))
12948 {
12949 rc = VINF_SUCCESS;
12950 Assert(cbOp == pDis->cbInstr);
12951 Log4Func(("Disas OpCode=%u CS:EIP %04x:%04RX64\n", pDis->pCurInstr->uOpcode, pMixedCtx->cs.Sel, pMixedCtx->rip));
12952 switch (pDis->pCurInstr->uOpcode)
12953 {
12954 case OP_CLI:
12955 {
12956 pMixedCtx->eflags.Bits.u1IF = 0;
12957 pMixedCtx->eflags.Bits.u1RF = 0;
12958 pMixedCtx->rip += pDis->cbInstr;
12959 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12960 if ( !fDbgStepping
12961 && pMixedCtx->eflags.Bits.u1TF)
12962 {
12963 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12964 AssertRCReturn(rc, rc);
12965 }
12966 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitCli);
12967 break;
12968 }
12969
12970 case OP_STI:
12971 {
12972 bool fOldIF = pMixedCtx->eflags.Bits.u1IF;
12973 pMixedCtx->eflags.Bits.u1IF = 1;
12974 pMixedCtx->eflags.Bits.u1RF = 0;
12975 pMixedCtx->rip += pDis->cbInstr;
12976 if (!fOldIF)
12977 {
12978 EMSetInhibitInterruptsPC(pVCpu, pMixedCtx->rip);
12979 Assert(VMCPU_FF_IS_PENDING(pVCpu, VMCPU_FF_INHIBIT_INTERRUPTS));
12980 }
12981 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12982 if ( !fDbgStepping
12983 && pMixedCtx->eflags.Bits.u1TF)
12984 {
12985 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
12986 AssertRCReturn(rc, rc);
12987 }
12988 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitSti);
12989 break;
12990 }
12991
12992 case OP_HLT:
12993 {
12994 rc = VINF_EM_HALT;
12995 pMixedCtx->rip += pDis->cbInstr;
12996 pMixedCtx->eflags.Bits.u1RF = 0;
12997 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP | HM_CHANGED_GUEST_RFLAGS);
12998 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitHlt);
12999 break;
13000 }
13001
13002 case OP_POPF:
13003 {
13004 Log4Func(("POPF CS:EIP %04x:%04RX64\n", pMixedCtx->cs.Sel, pMixedCtx->rip));
13005 uint32_t cbParm;
13006 uint32_t uMask;
13007 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13008 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13009 {
13010 cbParm = 4;
13011 uMask = 0xffffffff;
13012 }
13013 else
13014 {
13015 cbParm = 2;
13016 uMask = 0xffff;
13017 }
13018
13019 /* Get the stack pointer & pop the contents of the stack onto Eflags. */
13020 RTGCPTR GCPtrStack = 0;
13021 X86EFLAGS Eflags;
13022 Eflags.u32 = 0;
13023 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13024 &GCPtrStack);
13025 if (RT_SUCCESS(rc))
13026 {
13027 Assert(sizeof(Eflags.u32) >= cbParm);
13028 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u32, cbParm, PGMACCESSORIGIN_HM));
13029 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13030 }
13031 if (RT_FAILURE(rc))
13032 {
13033 rc = VERR_EM_INTERPRETER;
13034 break;
13035 }
13036 Log4Func(("POPF %#x -> %#RX64 mask=%#x RIP=%#RX64\n", Eflags.u, pMixedCtx->rsp, uMask, pMixedCtx->rip));
13037 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ~((X86_EFL_POPF_BITS & uMask) | X86_EFL_RF))
13038 | (Eflags.u32 & X86_EFL_POPF_BITS & uMask);
13039 pMixedCtx->esp += cbParm;
13040 pMixedCtx->esp &= uMask;
13041 pMixedCtx->rip += pDis->cbInstr;
13042 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13043 | HM_CHANGED_GUEST_RSP
13044 | HM_CHANGED_GUEST_RFLAGS);
13045 /* Generate a pending-debug exception when the guest stepping over POPF regardless of how
13046 POPF restores EFLAGS.TF. */
13047 if ( !fDbgStepping
13048 && fGstStepping)
13049 {
13050 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13051 AssertRCReturn(rc, rc);
13052 }
13053 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPopf);
13054 break;
13055 }
13056
13057 case OP_PUSHF:
13058 {
13059 uint32_t cbParm;
13060 uint32_t uMask;
13061 if (pDis->fPrefix & DISPREFIX_OPSIZE)
13062 {
13063 cbParm = 4;
13064 uMask = 0xffffffff;
13065 }
13066 else
13067 {
13068 cbParm = 2;
13069 uMask = 0xffff;
13070 }
13071
13072 /* Get the stack pointer & push the contents of eflags onto the stack. */
13073 RTGCPTR GCPtrStack = 0;
13074 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), (pMixedCtx->esp - cbParm) & uMask,
13075 SELMTOFLAT_FLAGS_CPL0, &GCPtrStack);
13076 if (RT_FAILURE(rc))
13077 {
13078 rc = VERR_EM_INTERPRETER;
13079 break;
13080 }
13081 X86EFLAGS Eflags = pMixedCtx->eflags;
13082 /* The RF & VM bits are cleared on image stored on stack; see Intel Instruction reference for PUSHF. */
13083 Eflags.Bits.u1RF = 0;
13084 Eflags.Bits.u1VM = 0;
13085
13086 rc = VBOXSTRICTRC_TODO(PGMPhysWrite(pVM, (RTGCPHYS)GCPtrStack, &Eflags.u, cbParm, PGMACCESSORIGIN_HM));
13087 if (RT_UNLIKELY(rc != VINF_SUCCESS))
13088 {
13089 AssertMsgFailed(("%Rrc\n", rc)); /** @todo allow strict return codes here */
13090 rc = VERR_EM_INTERPRETER;
13091 break;
13092 }
13093 Log4Func(("PUSHF %#x -> %#RGv\n", Eflags.u, GCPtrStack));
13094 pMixedCtx->esp -= cbParm;
13095 pMixedCtx->esp &= uMask;
13096 pMixedCtx->rip += pDis->cbInstr;
13097 pMixedCtx->eflags.Bits.u1RF = 0;
13098 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13099 | HM_CHANGED_GUEST_RSP
13100 | HM_CHANGED_GUEST_RFLAGS);
13101 if ( !fDbgStepping
13102 && pMixedCtx->eflags.Bits.u1TF)
13103 {
13104 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13105 AssertRCReturn(rc, rc);
13106 }
13107 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitPushf);
13108 break;
13109 }
13110
13111 case OP_IRET:
13112 {
13113 /** @todo Handle 32-bit operand sizes and check stack limits. See Intel
13114 * instruction reference. */
13115 RTGCPTR GCPtrStack = 0;
13116 uint32_t uMask = 0xffff;
13117 bool fGstStepping = RT_BOOL(pMixedCtx->eflags.Bits.u1TF);
13118 uint16_t aIretFrame[3];
13119 if (pDis->fPrefix & (DISPREFIX_OPSIZE | DISPREFIX_ADDRSIZE))
13120 {
13121 rc = VERR_EM_INTERPRETER;
13122 break;
13123 }
13124 rc = SELMToFlatEx(pVCpu, DISSELREG_SS, CPUMCTX2CORE(pMixedCtx), pMixedCtx->esp & uMask, SELMTOFLAT_FLAGS_CPL0,
13125 &GCPtrStack);
13126 if (RT_SUCCESS(rc))
13127 {
13128 rc = VBOXSTRICTRC_TODO(PGMPhysRead(pVM, (RTGCPHYS)GCPtrStack, &aIretFrame[0], sizeof(aIretFrame),
13129 PGMACCESSORIGIN_HM));
13130 AssertMsg(rc == VINF_SUCCESS, ("%Rrc\n", rc)); /** @todo allow strict return codes here */
13131 }
13132 if (RT_FAILURE(rc))
13133 {
13134 rc = VERR_EM_INTERPRETER;
13135 break;
13136 }
13137 pMixedCtx->eip = 0;
13138 pMixedCtx->ip = aIretFrame[0];
13139 pMixedCtx->cs.Sel = aIretFrame[1];
13140 pMixedCtx->cs.ValidSel = aIretFrame[1];
13141 pMixedCtx->cs.u64Base = (uint64_t)pMixedCtx->cs.Sel << 4;
13142 pMixedCtx->eflags.u32 = (pMixedCtx->eflags.u32 & ((UINT32_C(0xffff0000) | X86_EFL_1) & ~X86_EFL_RF))
13143 | (aIretFrame[2] & X86_EFL_POPF_BITS & uMask);
13144 pMixedCtx->sp += sizeof(aIretFrame);
13145 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RIP
13146 | HM_CHANGED_GUEST_CS
13147 | HM_CHANGED_GUEST_RSP
13148 | HM_CHANGED_GUEST_RFLAGS);
13149 /* Generate a pending-debug exception when stepping over IRET regardless of how IRET modifies EFLAGS.TF. */
13150 if ( !fDbgStepping
13151 && fGstStepping)
13152 {
13153 rc = hmR0VmxSetPendingDebugXcptVmcs(pVCpu, pMixedCtx);
13154 AssertRCReturn(rc, rc);
13155 }
13156 Log4Func(("IRET %#RX32 to %04x:%04x\n", GCPtrStack, pMixedCtx->cs.Sel, pMixedCtx->ip));
13157 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitIret);
13158 break;
13159 }
13160
13161 case OP_INT:
13162 {
13163 uint16_t uVector = pDis->Param1.uValue & 0xff;
13164 hmR0VmxSetPendingIntN(pVCpu, pMixedCtx, uVector, pDis->cbInstr);
13165 /* INT clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13166 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13167 break;
13168 }
13169
13170 case OP_INTO:
13171 {
13172 if (pMixedCtx->eflags.Bits.u1OF)
13173 {
13174 hmR0VmxSetPendingXcptOF(pVCpu, pMixedCtx, pDis->cbInstr);
13175 /* INTO clears EFLAGS.TF, we must not set any pending debug exceptions here. */
13176 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitInt);
13177 }
13178 else
13179 {
13180 pMixedCtx->eflags.Bits.u1RF = 0;
13181 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_GUEST_RFLAGS);
13182 }
13183 break;
13184 }
13185
13186 default:
13187 {
13188 pMixedCtx->eflags.Bits.u1RF = 0; /* This is correct most of the time... */
13189 VBOXSTRICTRC rc2 = EMInterpretInstructionDisasState(pVCpu, pDis, CPUMCTX2CORE(pMixedCtx), 0 /* pvFault */,
13190 EMCODETYPE_SUPERVISOR);
13191 rc = VBOXSTRICTRC_VAL(rc2);
13192 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13193 /** @todo We have to set pending-debug exceptions here when the guest is
13194 * single-stepping depending on the instruction that was interpreted. */
13195 Log4Func(("#GP rc=%Rrc\n", rc));
13196 break;
13197 }
13198 }
13199 }
13200 else
13201 rc = VERR_EM_INTERPRETER;
13202
13203 AssertMsg(rc == VINF_SUCCESS || rc == VERR_EM_INTERPRETER || rc == VINF_PGM_CHANGE_MODE || rc == VINF_EM_HALT,
13204 ("#GP Unexpected rc=%Rrc\n", rc));
13205 return rc;
13206}
13207
13208
13209/**
13210 * VM-exit exception handler wrapper for generic exceptions. Simply re-injects
13211 * the exception reported in the VMX transient structure back into the VM.
13212 *
13213 * @remarks Requires uExitIntInfo in the VMX transient structure to be
13214 * up-to-date.
13215 */
13216static int hmR0VmxExitXcptGeneric(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13217{
13218 RT_NOREF_PV(pMixedCtx);
13219 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13220#ifndef HMVMX_ALWAYS_TRAP_ALL_XCPTS
13221 AssertMsg(pVCpu->hm.s.fUsingDebugLoop || pVCpu->hm.s.vmx.RealMode.fRealOnV86Active,
13222 ("uVector=%#x u32XcptBitmap=%#X32\n",
13223 VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo), pVCpu->hm.s.vmx.u32XcptBitmap));
13224#endif
13225
13226 /* Re-inject the exception into the guest. This cannot be a double-fault condition which would have been handled in
13227 hmR0VmxCheckExitDueToEventDelivery(). */
13228 int rc = hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13229 rc |= hmR0VmxReadExitInstrLenVmcs(pVmxTransient);
13230 AssertRCReturn(rc, rc);
13231 Assert(ASMAtomicUoReadU32(&pVmxTransient->fVmcsFieldsRead) & HMVMX_READ_EXIT_INTERRUPTION_INFO);
13232
13233#ifdef DEBUG_ramshankar
13234 rc |= hmR0VmxImportGuestState(pVCpu, CPUMCTX_EXTRN_CS
13235 | CPUMCTX_EXTRN_RIP);
13236 uint8_t uVector = VMX_EXIT_INTERRUPTION_INFO_VECTOR(pVmxTransient->uExitIntInfo);
13237 Log(("hmR0VmxExitXcptGeneric: Reinjecting Xcpt. uVector=%#x cs:rip=%#04x:%#RX64\n", uVector, pCtx->cs.Sel, pCtx->rip));
13238#endif
13239
13240 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13241 pVmxTransient->cbInstr, pVmxTransient->uExitIntErrorCode, 0 /* GCPtrFaultAddress */);
13242 return VINF_SUCCESS;
13243}
13244
13245
13246/**
13247 * VM-exit exception handler for \#PF (Page-fault exception).
13248 */
13249static int hmR0VmxExitXcptPF(PVMCPU pVCpu, PCPUMCTX pMixedCtx, PVMXTRANSIENT pVmxTransient)
13250{
13251 HMVMX_VALIDATE_EXIT_XCPT_HANDLER_PARAMS();
13252 PVM pVM = pVCpu->CTX_SUFF(pVM);
13253 int rc = hmR0VmxReadExitQualificationVmcs(pVCpu, pVmxTransient);
13254 rc |= hmR0VmxReadExitIntInfoVmcs(pVmxTransient);
13255 rc |= hmR0VmxReadExitIntErrorCodeVmcs(pVmxTransient);
13256 AssertRCReturn(rc, rc);
13257
13258 if (!pVM->hm.s.fNestedPaging)
13259 { /* likely */ }
13260 else
13261 {
13262#if !defined(HMVMX_ALWAYS_TRAP_ALL_XCPTS) && !defined(HMVMX_ALWAYS_TRAP_PF)
13263 Assert(pVCpu->hm.s.fUsingDebugLoop);
13264#endif
13265 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory or vectoring #PF. */
13266 if (RT_LIKELY(!pVmxTransient->fVectoringDoublePF))
13267 {
13268 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13269 0 /* cbInstr */, pVmxTransient->uExitIntErrorCode, pVmxTransient->uExitQualification);
13270 }
13271 else
13272 {
13273 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13274 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13275 Log4Func(("Pending #DF due to vectoring #PF w/ NestedPaging\n"));
13276 }
13277 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13278 return rc;
13279 }
13280
13281 /* If it's a vectoring #PF, emulate injecting the original event injection as PGMTrap0eHandler() is incapable
13282 of differentiating between instruction emulation and event injection that caused a #PF. See @bugref{6607}. */
13283 if (pVmxTransient->fVectoringPF)
13284 {
13285 Assert(pVCpu->hm.s.Event.fPending);
13286 return VINF_EM_RAW_INJECT_TRPM_EVENT;
13287 }
13288
13289 rc = hmR0VmxImportGuestState(pVCpu, HMVMX_CPUMCTX_EXTRN_ALL);
13290 AssertRCReturn(rc, rc);
13291
13292 Log4Func(("#PF: cr2=%#RX64 cs:rip=%#04x:%#RX64 uErrCode %#RX32 cr3=%#RX64\n", pVmxTransient->uExitQualification,
13293 pMixedCtx->cs.Sel, pMixedCtx->rip, pVmxTransient->uExitIntErrorCode, pMixedCtx->cr3));
13294
13295 TRPMAssertXcptPF(pVCpu, pVmxTransient->uExitQualification, (RTGCUINT)pVmxTransient->uExitIntErrorCode);
13296 rc = PGMTrap0eHandler(pVCpu, pVmxTransient->uExitIntErrorCode, CPUMCTX2CORE(pMixedCtx),
13297 (RTGCPTR)pVmxTransient->uExitQualification);
13298
13299 Log4Func(("#PF: rc=%Rrc\n", rc));
13300 if (rc == VINF_SUCCESS)
13301 {
13302 /*
13303 * This is typically a shadow page table sync or a MMIO instruction. But we may have
13304 * emulated something like LTR or a far jump. Any part of the CPU context may have changed.
13305 */
13306 ASMAtomicUoOrU64(&pVCpu->hm.s.fCtxChanged, HM_CHANGED_ALL_GUEST);
13307 TRPMResetTrap(pVCpu);
13308 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPF);
13309 return rc;
13310 }
13311
13312 if (rc == VINF_EM_RAW_GUEST_TRAP)
13313 {
13314 if (!pVmxTransient->fVectoringDoublePF)
13315 {
13316 /* It's a guest page fault and needs to be reflected to the guest. */
13317 uint32_t uGstErrorCode = TRPMGetErrorCode(pVCpu);
13318 TRPMResetTrap(pVCpu);
13319 pVCpu->hm.s.Event.fPending = false; /* In case it's a contributory #PF. */
13320 hmR0VmxSetPendingEvent(pVCpu, VMX_VMCS_CTRL_ENTRY_IRQ_INFO_FROM_EXIT_INT_INFO(pVmxTransient->uExitIntInfo),
13321 0 /* cbInstr */, uGstErrorCode, pVmxTransient->uExitQualification);
13322 }
13323 else
13324 {
13325 /* A guest page-fault occurred during delivery of a page-fault. Inject #DF. */
13326 TRPMResetTrap(pVCpu);
13327 pVCpu->hm.s.Event.fPending = false; /* Clear pending #PF to replace it with #DF. */
13328 hmR0VmxSetPendingXcptDF(pVCpu, pMixedCtx);
13329 Log4Func(("#PF: Pending #DF due to vectoring #PF\n"));
13330 }
13331
13332 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitGuestPF);
13333 return VINF_SUCCESS;
13334 }
13335
13336 TRPMResetTrap(pVCpu);
13337 STAM_COUNTER_INC(&pVCpu->hm.s.StatExitShadowPFEM);
13338 return rc;
13339}
13340
13341/** @} */
13342
Note: See TracBrowser for help on using the repository browser.

© 2024 Oracle Support Privacy / Do Not Sell My Info Terms of Use Trademark Policy Automated Access Etiquette